From fb33bd6916a9d100a3515856b12e40cfc3c392a1 Mon Sep 17 00:00:00 2001 From: "Dan P." Date: Thu, 9 Jan 2025 10:49:55 -0500 Subject: [PATCH] [dev-v2.11] rancher-monitoring 106.0.0+up61.3.2 create (#4932) --- ...ancher-monitoring-crd-106.0.0+up61.3.2.tgz | Bin 0 -> 331974 bytes .../rancher-monitoring-106.0.0+up61.3.2.tgz | Bin 0 -> 486460 bytes .../106.0.0+up61.3.2/Chart.yaml | 10 + .../106.0.0+up61.3.2/README.md | 24 + .../106.0.0+up61.3.2/files/crd-manifest.tgz | Bin 0 -> 331362 bytes .../106.0.0+up61.3.2/templates/_helpers.tpl | 30 + .../106.0.0+up61.3.2/templates/jobs.yaml | 102 + .../106.0.0+up61.3.2/templates/manifest.yaml | 8 + .../106.0.0+up61.3.2/templates/rbac.yaml | 76 + .../templates/validate-psp-install.yaml | 7 + .../106.0.0+up61.3.2/values.yaml | 17 + .../106.0.0+up61.3.2/.editorconfig | 5 + .../106.0.0+up61.3.2/.helmignore | 29 + .../106.0.0+up61.3.2/CHANGELOG.md | 47 + .../106.0.0+up61.3.2/CONTRIBUTING.md | 12 + .../106.0.0+up61.3.2/Chart.yaml | 126 + .../106.0.0+up61.3.2/README.md | 1140 ++++ .../106.0.0+up61.3.2/app-README.md | 46 + .../charts/grafana/.helmignore | 23 + .../charts/grafana/Chart.yaml | 41 + .../106.0.0+up61.3.2/charts/grafana/README.md | 778 +++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 55 + .../charts/grafana/templates/_config.tpl | 172 + .../charts/grafana/templates/_helpers.tpl | 305 + .../charts/grafana/templates/_pod.tpl | 1306 ++++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 24 + .../grafana/templates/configSecret.yaml | 43 + .../configmap-dashboard-provider.yaml | 15 + .../charts/grafana/templates/configmap.yaml | 20 + .../templates/dashboards-json-configmap.yaml | 38 + .../charts/grafana/templates/deployment.yaml | 53 + .../grafana/templates/extra-manifests.yaml | 4 + .../grafana/templates/headless-service.yaml | 22 + .../charts/grafana/templates/hpa.yaml | 52 + .../templates/image-renderer-deployment.yaml | 199 + .../grafana/templates/image-renderer-hpa.yaml | 47 + .../image-renderer-network-policy.yaml | 79 + .../templates/image-renderer-service.yaml | 31 + .../image-renderer-servicemonitor.yaml | 48 + .../charts/grafana/templates/ingress.yaml | 78 + .../grafana/templates/networkpolicy.yaml | 61 + .../grafana/templates/nginx-config.yaml | 94 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 45 + .../charts/grafana/templates/pvc.yaml | 41 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 25 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 16 + .../charts/grafana/templates/service.yaml | 67 + .../grafana/templates/serviceaccount.yaml | 17 + .../grafana/templates/servicemonitor.yaml | 66 + .../charts/grafana/templates/statefulset.yaml | 58 + .../templates/tests/test-configmap.yaml | 20 + .../tests/test-podsecuritypolicy.yaml | 32 + .../grafana/templates/tests/test-role.yaml | 17 + .../templates/tests/test-rolebinding.yaml | 20 + .../templates/tests/test-serviceaccount.yaml | 12 + .../charts/grafana/templates/tests/test.yaml | 53 + .../charts/grafana/values.yaml | 1365 ++++ .../charts/hardenedKubelet/.helmignore | 23 + .../charts/hardenedKubelet/Chart.yaml | 15 + .../charts/hardenedKubelet/README.md | 90 + .../hardenedKubelet/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedKubelet/values.yaml | 166 + .../charts/hardenedNodeExporter/.helmignore | 23 + .../charts/hardenedNodeExporter/Chart.yaml | 15 + .../charts/hardenedNodeExporter/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/hardenedNodeExporter/values.yaml | 166 + .../charts/k3sServer/.helmignore | 23 + .../charts/k3sServer/Chart.yaml | 15 + .../charts/k3sServer/README.md | 90 + .../charts/k3sServer/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../k3sServer/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../k3sServer/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/k3sServer/values.yaml | 166 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 32 + .../charts/kube-state-metrics/README.md | 85 + .../kube-state-metrics/templates/NOTES.txt | 23 + .../kube-state-metrics/templates/_helpers.tpl | 196 + .../templates/ciliumnetworkpolicy.yaml | 33 + .../templates/clusterrolebinding.yaml | 20 + .../templates/crs-configmap.yaml | 16 + .../templates/deployment.yaml | 313 + .../templates/extra-manifests.yaml | 4 + .../templates/kubeconfig-secret.yaml | 12 + .../templates/networkpolicy.yaml | 43 + .../kube-state-metrics/templates/pdb.yaml | 18 + .../templates/podsecuritypolicy.yaml | 39 + .../templates/psp-clusterrole.yaml | 19 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/rbac-configmap.yaml | 22 + .../kube-state-metrics/templates/role.yaml | 215 + .../templates/rolebinding.yaml | 24 + .../kube-state-metrics/templates/service.yaml | 53 + .../templates/serviceaccount.yaml | 18 + .../templates/servicemonitor.yaml | 126 + .../templates/stsdiscovery-role.yaml | 26 + .../templates/stsdiscovery-rolebinding.yaml | 17 + .../templates/verticalpodautoscaler.yaml | 44 + .../charts/kube-state-metrics/values.yaml | 523 ++ .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 15 + .../charts/kubeAdmControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../kubeAdmControllerManager/values.yaml | 166 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 15 + .../charts/kubeAdmEtcd/README.md | 90 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmEtcd/values.yaml | 166 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 15 + .../charts/kubeAdmProxy/README.md | 90 + .../kubeAdmProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmProxy/values.yaml | 166 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 15 + .../charts/kubeAdmScheduler/README.md | 90 + .../kubeAdmScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/kubeAdmScheduler/values.yaml | 166 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 27 + .../charts/prometheus-adapter/README.md | 160 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 113 + .../templates/certmanager.yaml | 76 + .../cluster-role-binding-auth-delegator.yaml | 20 + .../cluster-role-binding-auth-reader.yaml | 20 + .../cluster-role-binding-resource-reader.yaml | 20 + .../cluster-role-resource-reader.yaml | 24 + .../templates/configmap.yaml | 97 + .../templates/custom-metrics-apiservice.yaml | 34 + ...stom-metrics-cluster-role-binding-hpa.yaml | 24 + .../custom-metrics-cluster-role.yaml | 17 + .../templates/deployment.yaml | 151 + .../external-metrics-apiservice.yaml | 34 + ...rnal-metrics-cluster-role-binding-hpa.yaml | 20 + .../external-metrics-cluster-role.yaml | 20 + .../prometheus-adapter/templates/pdb.yaml | 23 + .../prometheus-adapter/templates/psp.yaml | 66 + .../resource-metrics-apiservice.yaml | 34 + ...resource-metrics-cluster-role-binding.yaml | 20 + .../resource-metrics-cluster-role.yaml | 23 + .../templates/role-binding-auth-reader.yaml | 21 + .../prometheus-adapter/templates/secret.yaml | 17 + .../prometheus-adapter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 18 + .../charts/prometheus-adapter/values.yaml | 292 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 25 + .../charts/prometheus-node-exporter/README.md | 96 + .../templates/NOTES.txt | 29 + .../templates/_helpers.tpl | 236 + .../templates/clusterrole.yaml | 19 + .../templates/clusterrolebinding.yaml | 20 + .../templates/daemonset.yaml | 312 + .../templates/endpoints.yaml | 18 + .../templates/extra-manifests.yaml | 4 + .../templates/networkpolicy.yaml | 23 + .../templates/podmonitor.yaml | 91 + .../templates/psp-clusterrole.yaml | 14 + .../templates/psp-clusterrolebinding.yaml | 16 + .../templates/psp.yaml | 49 + .../templates/rbac-configmap.yaml | 16 + .../templates/service.yaml | 35 + .../templates/serviceaccount.yaml | 18 + .../templates/servicemonitor.yaml | 71 + .../templates/verticalpodautoscaler.yaml | 40 + .../prometheus-node-exporter/values.yaml | 539 ++ .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 15 + .../charts/rke2ControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2ControllerManager/values.yaml | 166 + .../charts/rke2Etcd/.helmignore | 23 + .../charts/rke2Etcd/Chart.yaml | 15 + .../charts/rke2Etcd/README.md | 90 + .../charts/rke2Etcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Etcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Etcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Etcd/values.yaml | 166 + .../charts/rke2IngressNginx/.helmignore | 23 + .../charts/rke2IngressNginx/Chart.yaml | 15 + .../charts/rke2IngressNginx/README.md | 90 + .../rke2IngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2IngressNginx/values.yaml | 166 + .../charts/rke2Proxy/.helmignore | 23 + .../charts/rke2Proxy/Chart.yaml | 15 + .../charts/rke2Proxy/README.md | 90 + .../charts/rke2Proxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rke2Proxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rke2Proxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Proxy/values.yaml | 166 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 15 + .../charts/rke2Scheduler/README.md | 90 + .../rke2Scheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rke2Scheduler/values.yaml | 166 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 15 + .../charts/rkeControllerManager/README.md | 90 + .../templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeControllerManager/values.yaml | 166 + .../charts/rkeEtcd/.helmignore | 23 + .../charts/rkeEtcd/Chart.yaml | 15 + .../106.0.0+up61.3.2/charts/rkeEtcd/README.md | 90 + .../charts/rkeEtcd/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeEtcd/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeEtcd/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeEtcd/values.yaml | 166 + .../charts/rkeIngressNginx/.helmignore | 23 + .../charts/rkeIngressNginx/Chart.yaml | 15 + .../charts/rkeIngressNginx/README.md | 90 + .../rkeIngressNginx/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeIngressNginx/values.yaml | 166 + .../charts/rkeProxy/.helmignore | 23 + .../charts/rkeProxy/Chart.yaml | 15 + .../charts/rkeProxy/README.md | 90 + .../charts/rkeProxy/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../rkeProxy/templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../rkeProxy/templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeProxy/values.yaml | 166 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 15 + .../charts/rkeScheduler/README.md | 90 + .../rkeScheduler/templates/_helpers.tpl | 170 + .../templates/pushprox-clients-rbac.yaml | 97 + .../templates/pushprox-clients.yaml | 157 + .../templates/pushprox-proxy-rbac.yaml | 68 + .../templates/pushprox-proxy.yaml | 57 + .../templates/pushprox-servicemonitor.yaml | 45 + .../templates/validate-install-crd.yaml | 14 + .../templates/validate-psp-install.yaml | 7 + .../charts/rkeScheduler/values.yaml | 166 + .../charts/windowsExporter/.helmignore | 21 + .../charts/windowsExporter/Chart.yaml | 24 + .../charts/windowsExporter/README.md | 42 + .../scripts/configure-firewall.ps1 | 31 + .../windowsExporter/templates/_helpers.tpl | 216 + .../windowsExporter/templates/config.yaml | 14 + .../windowsExporter/templates/daemonset.yaml | 200 + .../windowsExporter/templates/podmonitor.yaml | 91 + .../templates/scriptConfig.yaml | 14 + .../windowsExporter/templates/service.yaml | 32 + .../templates/serviceaccount.yaml | 17 + .../templates/servicemonitor.yaml | 75 + .../charts/windowsExporter/values.yaml | 367 ++ .../files/ingress-nginx/nginx.json | 1445 +++++ .../request-handling-performance.json | 963 +++ .../cluster/rancher-cluster-nodes.json | 793 +++ .../rancher/cluster/rancher-cluster.json | 776 +++ .../files/rancher/fleet/bundle.json | 246 + .../files/rancher/fleet/bundledeployment.json | 219 + .../files/rancher/fleet/cluster.json | 484 ++ .../files/rancher/fleet/clustergroup.json | 468 ++ .../rancher/fleet/controller-runtime.json | 454 ++ .../files/rancher/fleet/gitrepo.json | 325 + .../rancher/home/rancher-default-home.json | 1290 ++++ .../files/rancher/k8s/rancher-etcd-nodes.json | 687 ++ .../files/rancher/k8s/rancher-etcd.json | 669 ++ .../k8s/rancher-k8s-components-nodes.json | 527 ++ .../rancher/k8s/rancher-k8s-components.json | 519 ++ .../files/rancher/logging/fluentbit.json | 760 +++ .../files/rancher/logging/fluentd.json | 3221 ++++++++++ .../rancher/nodes/rancher-node-detail.json | 805 +++ .../files/rancher/nodes/rancher-node.json | 792 +++ .../performance/performance-debugging.json | 1652 +++++ .../rancher/pods/rancher-pod-containers.json | 636 ++ .../files/rancher/pods/rancher-pod.json | 636 ++ .../workloads/rancher-workload-pods.json | 652 ++ .../rancher/workloads/rancher-workload.json | 652 ++ .../delete-workloads-with-old-labels.sh | 14 + .../106.0.0+up61.3.2/templates/NOTES.txt | 4 + .../106.0.0+up61.3.2/templates/_helpers.tpl | 472 ++ .../templates/alertmanager/alertmanager.yaml | 195 + .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 78 + .../alertmanager/ingressperreplica.yaml | 67 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 23 + .../alertmanager/psp-rolebinding.yaml | 20 + .../templates/alertmanager/psp.yaml | 47 + .../templates/alertmanager/secret.yaml | 37 + .../templates/alertmanager/service.yaml | 72 + .../alertmanager/serviceaccount.yaml | 21 + .../alertmanager/servicemonitor.yaml | 84 + .../alertmanager/serviceperreplica.yaml | 49 + .../templates/exporters/core-dns/service.yaml | 28 + .../exporters/core-dns/servicemonitor.yaml | 58 + .../kube-api-server/servicemonitor.yaml | 57 + .../kube-controller-manager/endpoints.yaml | 22 + .../kube-controller-manager/service.yaml | 33 + .../servicemonitor.yaml | 69 + .../templates/exporters/kube-dns/service.yaml | 32 + .../exporters/kube-dns/servicemonitor.yaml | 71 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 31 + .../exporters/kube-etcd/servicemonitor.yaml | 75 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 31 + .../exporters/kube-proxy/servicemonitor.yaml | 63 + .../exporters/kube-scheduler/endpoints.yaml | 22 + .../exporters/kube-scheduler/service.yaml | 33 + .../kube-scheduler/servicemonitor.yaml | 69 + .../kube-state-metrics/validate.yaml | 7 + .../exporters/kubelet/servicemonitor.yaml | 246 + .../exporters/node-exporter/validate.yaml | 3 + .../templates/extra-objects.yaml | 4 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 81 + .../alertmanager-overview.yaml | 616 ++ .../grafana/dashboards-1.14/apiserver.yaml | 1772 ++++++ .../dashboards-1.14/cluster-total.yaml | 1882 ++++++ .../dashboards-1.14/controller-manager.yaml | 1196 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1229 ++++ .../dashboards-1.14/grafana-overview.yaml | 635 ++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1534 +++++ .../k8s-resources-cluster.yaml | 3088 +++++++++ .../k8s-resources-multicluster.yaml | 24 + .../k8s-resources-namespace.yaml | 2797 +++++++++ .../dashboards-1.14/k8s-resources-node.yaml | 1026 +++ .../dashboards-1.14/k8s-resources-pod.yaml | 2469 ++++++++ .../k8s-resources-windows-cluster.yaml | 24 + .../k8s-resources-windows-namespace.yaml | 24 + .../k8s-resources-windows-pod.yaml | 24 + .../k8s-resources-workload.yaml | 2024 ++++++ .../k8s-resources-workloads-namespace.yaml | 2189 +++++++ .../k8s-windows-cluster-rsrc-use.yaml | 24 + .../k8s-windows-node-rsrc-use.yaml | 24 + .../grafana/dashboards-1.14/kubelet.yaml | 2256 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1464 +++++ .../namespace-by-workload.yaml | 1736 +++++ .../node-cluster-rsrc-use.yaml | 1063 ++++ .../dashboards-1.14/node-rsrc-use.yaml | 1089 ++++ .../grafana/dashboards-1.14/nodes-darwin.yaml | 1073 ++++ .../grafana/dashboards-1.14/nodes.yaml | 1066 ++++ .../persistentvolumesusage.yaml | 587 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1228 ++++ .../prometheus-remote-write.yaml | 1674 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1235 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1276 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1118 ++++ .../dashboards-1.14/workload-total.yaml | 1438 +++++ .../templates/grafana/namespaces.yaml | 13 + .../_prometheus-operator.tpl | 7 + .../_prometheus-operator-webhook.tpl | 6 + .../deployment/deployment.yaml | 143 + .../admission-webhooks/deployment/pdb.yaml | 15 + .../deployment/service.yaml | 62 + .../deployment/serviceaccount.yaml | 15 + .../ciliumnetworkpolicy-createSecret.yaml | 36 + .../ciliumnetworkpolicy-patchWebhook.yaml | 36 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 73 + .../job-patch/job-patchWebhook.yaml | 74 + .../job-patch/networkpolicy-createSecret.yaml | 33 + .../job-patch/networkpolicy-patchWebhook.yaml | 33 + .../admission-webhooks/job-patch/psp.yaml | 47 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 18 + .../mutatingWebhookConfiguration.yaml | 81 + .../validatingWebhookConfiguration.yaml | 81 + .../aggregate-clusterroles.yaml | 29 + .../prometheus-operator/certmanager.yaml | 55 + .../ciliumnetworkpolicy.yaml | 40 + .../prometheus-operator/clusterrole.yaml | 109 + .../clusterrolebinding.yaml | 16 + .../prometheus-operator/deployment.yaml | 207 + .../prometheus-operator/networkpolicy.yaml | 29 + .../prometheus-operator/psp-clusterrole.yaml | 21 + .../psp-clusterrolebinding.yaml | 18 + .../templates/prometheus-operator/psp.yaml | 46 + .../prometheus-operator/service.yaml | 61 + .../prometheus-operator/serviceaccount.yaml | 14 + .../prometheus-operator/servicemonitor.yaml | 57 + .../verticalpodautoscaler.yaml | 40 + .../templates/prometheus/_rules.tpl | 44 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 43 + .../prometheus/additionalScrapeConfigs.yaml | 20 + .../prometheus/ciliumnetworkpolicy.yaml | 27 + .../templates/prometheus/clusterrole.yaml | 43 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 77 + .../prometheus/ingressThanosSidecar.yaml | 77 + .../prometheus/ingressperreplica.yaml | 67 + .../templates/prometheus/networkpolicy.yaml | 34 + .../templates/prometheus/nginx-config.yaml | 68 + .../prometheus/podDisruptionBudget.yaml | 25 + .../templates/prometheus/podmonitors.yaml | 38 + .../templates/prometheus/prometheus.yaml | 481 ++ .../templates/prometheus/psp-clusterrole.yaml | 22 + .../prometheus/psp-clusterrolebinding.yaml | 19 + .../templates/prometheus/psp.yaml | 58 + .../rules-1.14/alertmanager.rules.yaml | 305 + .../rules-1.14/config-reloaders.yaml | 57 + .../templates/prometheus/rules-1.14/etcd.yaml | 461 ++ .../prometheus/rules-1.14/general.rules.yaml | 125 + ...les.container_cpu_usage_seconds_total.yaml | 43 + .../k8s.rules.container_memory_cache.yaml | 42 + .../k8s.rules.container_memory_rss.yaml | 42 + .../k8s.rules.container_memory_swap.yaml | 42 + ...es.container_memory_working_set_bytes.yaml | 42 + .../k8s.rules.container_resource.yaml | 168 + .../rules-1.14/k8s.rules.pod_owner.yaml | 107 + .../prometheus/rules-1.14/k8s.rules.yaml | 237 + .../kube-apiserver-availability.rules.yaml | 273 + .../kube-apiserver-burnrate.rules.yaml | 440 ++ .../kube-apiserver-histogram.rules.yaml | 53 + .../rules-1.14/kube-apiserver-slos.yaml | 159 + .../kube-prometheus-general.rules.yaml | 49 + .../kube-prometheus-node-recording.rules.yaml | 93 + .../rules-1.14/kube-scheduler.rules.yaml | 135 + .../rules-1.14/kube-state-metrics.yaml | 152 + .../prometheus/rules-1.14/kubelet.rules.yaml | 65 + .../rules-1.14/kubernetes-apps.yaml | 568 ++ .../rules-1.14/kubernetes-resources.yaml | 282 + .../rules-1.14/kubernetes-storage.yaml | 216 + .../kubernetes-system-apiserver.yaml | 193 + .../kubernetes-system-controller-manager.yaml | 57 + .../kubernetes-system-kube-proxy.yaml | 52 + .../rules-1.14/kubernetes-system-kubelet.yaml | 379 ++ .../kubernetes-system-scheduler.yaml | 54 + .../rules-1.14/kubernetes-system.yaml | 87 + .../rules-1.14/node-exporter.rules.yaml | 188 + .../prometheus/rules-1.14/node-exporter.yaml | 801 +++ .../prometheus/rules-1.14/node-network.yaml | 55 + .../prometheus/rules-1.14/node.rules.yaml | 109 + .../rules-1.14/prometheus-operator.yaml | 253 + .../prometheus/rules-1.14/prometheus.yaml | 735 +++ .../rules-1.14/windows.node.rules.yaml | 301 + .../rules-1.14/windows.pod.rules.yaml | 158 + .../templates/prometheus/secret.yaml | 15 + .../templates/prometheus/service.yaml | 84 + .../prometheus/serviceThanosSidecar.yaml | 43 + .../serviceThanosSidecarExternal.yaml | 46 + .../templates/prometheus/serviceaccount.yaml | 21 + .../templates/prometheus/servicemonitor.yaml | 97 + .../servicemonitorThanosSidecar.yaml | 55 + .../templates/prometheus/servicemonitors.yaml | 47 + .../prometheus/serviceperreplica.yaml | 58 + .../rancher-monitoring/clusterrole.yaml | 135 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../addons/ingress-nginx-dashboard.yaml | 18 + .../rancher/cluster-dashboards.yaml | 17 + .../dashboards/rancher/default-dashboard.yaml | 17 + .../dashboards/rancher/fleet-dashboards.yaml | 17 + .../rancher/fluentbit-dashboard.yaml | 17 + .../dashboards/rancher/fluentd-dashboard.yaml | 17 + .../dashboards/rancher/k8s-dashboards.yaml | 31 + .../dashboards/rancher/nodes-dashboards.yaml | 17 + .../rancher/performance-dashboards.yaml | 18 + .../dashboards/rancher/pods-dashboards.yaml | 17 + .../rancher/workload-dashboards.yaml | 17 + .../exporters/fleet/servicemonitor.yaml | 53 + .../ingress-nginx/network-policy.yaml | 19 + .../exporters/ingress-nginx/service.yaml | 27 + .../ingress-nginx/servicemonitor.yaml | 49 + .../exporters/rancher/servicemonitor.yaml | 58 + .../rancher-monitoring/hardened.yaml | 91 + .../rancher-monitoring/upgrade/configmap.yaml | 13 + .../rancher-monitoring/upgrade/job.yaml | 46 + .../rancher-monitoring/upgrade/rbac.yaml | 86 + .../templates/thanos-ruler/extrasecret.yaml | 20 + .../templates/thanos-ruler/ingress.yaml | 77 + .../thanos-ruler/podDisruptionBudget.yaml | 21 + .../templates/thanos-ruler/ruler.yaml | 200 + .../templates/thanos-ruler/secret.yaml | 26 + .../templates/thanos-ruler/service.yaml | 57 + .../thanos-ruler/serviceaccount.yaml | 20 + .../thanos-ruler/servicemonitor.yaml | 82 + .../templates/validate-install-crd.yaml | 23 + .../templates/validate-psp-install.yaml | 2 + .../106.0.0+up61.3.2/values.yaml | 5567 +++++++++++++++++ index.yaml | 144 + .../generated-changes/patch/values.yaml.patch | 4 +- .../rancher-monitoring/package.yaml | 2 +- release.yaml | 2 + 591 files changed, 109791 insertions(+), 3 deletions(-) create mode 100644 assets/rancher-monitoring-crd/rancher-monitoring-crd-106.0.0+up61.3.2.tgz create mode 100644 assets/rancher-monitoring/rancher-monitoring-106.0.0+up61.3.2.tgz create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/Chart.yaml create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/README.md create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/files/crd-manifest.tgz create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/jobs.yaml create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/manifest.yaml create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/rbac.yaml create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring-crd/106.0.0+up61.3.2/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/.editorconfig create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/CHANGELOG.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/CONTRIBUTING.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/app-README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/dashboards/custom-dashboard.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_config.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_pod.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configSecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/headless-service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/hpa.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-hpa.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-network-policy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/ingress.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/nginx-config.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/poddisruptionbudget.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/pvc.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret-env.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/statefulset.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/crs-configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/certmanager.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/pdb.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/extra-manifests.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/rbac-configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/.helmignore create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/Chart.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/README.md create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/scripts/configure-firewall.ps1 create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/config.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/daemonset.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/podmonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/scriptConfig.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/values.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/nginx.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/request-handling-performance.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster-nodes.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundle.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundledeployment.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/cluster.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/clustergroup.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/controller-runtime.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/gitrepo.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/home/rancher-default-home.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd-nodes.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components-nodes.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentbit.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentd.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node-detail.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/performance/performance-debugging.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod-containers.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload-pods.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload.json create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/NOTES.txt create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/_helpers.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/alertmanager.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/extrasecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingress.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/endpoints.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/endpoints.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/endpoints.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-state-metrics/validate.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kubelet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/node-exporter/validate.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/extra-objects.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmap-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmaps-datasources.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/grafana-overview.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/namespaces.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/_prometheus-operator.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/aggregate-clusterroles.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/deployment.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/verticalpodautoscaler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/_rules.tpl create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalPrometheusRules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalScrapeConfigs.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ciliumnetworkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/csi-secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/extrasecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingress.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressperreplica.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/networkpolicy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/nginx-config.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podmonitors.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/prometheus.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrolebinding.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/config-reloaders.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/etcd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/general.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-network.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.node.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.pod.rules.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecarExternal.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitorThanosSidecar.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitors.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceperreplica.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/clusterrole.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/config-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboard-role.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentbit-dashboard.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentd-dashboard.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/hardened.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/configmap.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/job.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/rbac.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/extrasecret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ingress.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/podDisruptionBudget.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ruler.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/secret.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/service.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/serviceaccount.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/servicemonitor.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-install-crd.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-psp-install.yaml create mode 100644 charts/rancher-monitoring/106.0.0+up61.3.2/values.yaml diff --git a/assets/rancher-monitoring-crd/rancher-monitoring-crd-106.0.0+up61.3.2.tgz b/assets/rancher-monitoring-crd/rancher-monitoring-crd-106.0.0+up61.3.2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b3a27ce48df85e933069c0727b6b2ff5fe152647 GIT binary patch literal 331974 zcmV(*K;FL}iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMR7Jd|JiFis-7WQioVC0oY6lzmHx?8$D-+!)5pn1!sV$eQfg zLs?R`kPwn&$u4W8Bw0dHgz*0i+P=?sdw%cpKF|C9-{*Y#_FkcR^54HxP$` z!a#Xor}O7chx5YWLAT%i4htedC=P@WzyD6eZ#FD76oWVmVsQvGN*?f#q=thy7#4xS zZ@=rHaCj&Zi39M?AfREai31$b*xx}UO8uu(;Jq-QJOIUDkOG9M*KJWFDot~^^uMfh$7;G%L9&3Bo5r}+%Xut#2bT72pk^kB@giMP*Wq^pibL3upkDFBd);fyHbnc zoIxa#8UXN6Cwag_TvSq2f|?qIhJz*`5`^K=Sb4z5kD3~fMuON~y9ij>{NJO{|780E z{r@fHq5gzoYR2#=Bmf+Q$b1z(Gd@ z3Iup|w95{@qh5B{UH#$#Abx(^fCxuG^z4q4*wNl^4*PfY^805PECPjh1bB|(#CZNk z2tXA4AD*ds0doWjj`qX>CIk!yjl~leBPLA!trvzQ;P4<8@I)Yy02B@fcGm^qp$HUy z+vFh;D8hMwXa{g2)I<;m5Q+BO?f%Ia5Y3q<8taNgL*Y08h9)41pa2f$48jRW5KbJ& zJEL*n&lstN#fW77%;4=&2yqnh6ZRecLI78gXv%(sKxiB>q}sO7^1I&>J3vQA5Qdir zc+Q|rb_pPMqcKE$L`%kV(wzWB^8CuJKOh6QYyWqxjCVZ`<`JN}d<0Pt_PAmbXhC72s8KcqopJVLr zV`exID-Ve8cEZpoqT>XyKamaDUG2`=h&BZGeGCA0B61>u-#GRsA%-a6=SN)1ZgjBy zt>}+X9`N&%|A%2b)ae_SewxvcfJ7Q@OIaS!aXf>@8)88mh{FE_2SdBLK~Zq|pSmD` zm;(YOhI9U<4+8t;Rr#kk003dmXn;ozgF$*BP)^%522oK_9)(|k++5)ZEC9g(VjvzS z2E)RC9d>}?Kv`)30^c1+qZ|=VL@=Th5sn}ZFN$~a7Il!727CcL6bnGkdwT#7jXyv~ zbf+*pl9&S_iMj_dcKi+r`1*by-x1g^!wSE%Hcd1NT|o{~WRY8O1-t{XK$r~EeKW~sfvWnlohCm?H{{pwPW99$d zZV3zinE3xRzw;Ae|G)D)yS)8>((h;z*ZzN+-`Noe0D&N!P{du1LgOKhXaWkZL^Sz- z=x_e(R{rZ&{`a_*|8%DHe`fp3{QtWV_@CDR{_6ZkN>bvt^==>KKyFZCbm0EPYUt^Y|$O8oZzhn%$JfA#<0(tg(eZBauBcxN;g;l1s> zMO}|?SL1(JhMkiXV>A-{yS8CB5%}x;>N_x&fFwo&5CDonoWi0BnC;KDU%~Tu5JjXG zw@cdgDK8F9z{0>?C>Q~UN4tIN+&<_)Y)2y00N?>)9dPo1mB3EbLqG%&a02nfKNw2K!y=h*qLs$qObNhI^1 z+0Q>OX=gKwc}O^bc<8s~8KU7PyE}O&i~3Km@LSaagGG2Ckf77g*GHg8B-+#P+rUX2 z42p#A+!gx<;f%)N&wzMOG}iUkP8~yyUpowSG=Fsbs9J23NVH_~fPgWvR}JMQKn(ym zP>+~`{qZ;fgF%9BAPNsf{xE*uLG=kZ{5H72PA?SY^mEqahCs;!65rDzH|TkJKvqUZ zN`@K$9C16k^)EvAE!j3kqw(9}+ICP=Nb=pcZ=_pD9eTMrI|VL`mW&!*}>?GFE+ zYkwL4c|ef}q924{aF}lgZvSLF_*eO#toU#4|4B;9NdNc#&%dSd?m7V{JRpb!QGg&4 zjdmqq0G>Y!5_kZfpS&**z(f2eBm{_WU(DeBXZ*@HU=$h;2>yDjM9olV*X7_uwV^nG z=U2ZY_75J1N9a37M<@ab@R&M-03M2U0`cFk0Kw59ZhJCks0X-xA^0EA0BVLh-(xLN zz;N3U0dNG4NHkz*Vu%9}C;;z_zyUC#=h>~}0~jnC27<&`NtEXYBX$kjA3gu4<^Koz zZ+ucsQ~#u>8~k4}$6x5bwB$eD|Cji0{pa7)c>n0OyV|#lfxDX=?+nEQh<~_APr!k2 zz`^TxZ_`ur@&eju&mEonNneOTl?pYr>5m>aC>H0ujTnkV0%*r?__wbWXMlqkaHnAP&B%Z_0Ez{FB2x_5 zz05*<#G<$R9H1~)k)PQJo(EwBJn`FKZo>d#zg&m;_WJEU44{j4*k%kd6EY-by+jNs zF91A`z-^x*?f$9(_mdoO-_PFQfF}a){QHtX@AQ6SE*=fwKs;bi48%n-SP%zdJwP** zD+=w2(n5epI8JFhR9S#J1}NaLA8wv7Pb&aG zpqzxZr3?kEwwJKIjiFdH0R{g-h!X34%j7;lzj>+!iGkf=8o~I57kc zM*wkR(sHuA+mA4`8!_yM$exgpkvMTeN>)e&z!6~1y8_rgdj=f9A6o;3#=1d~NH4$< zi*^I-f3H09%~{$LrSV4)_M+4fz?gs{?j-wf=XX#H2I=)(rrRWJs}pgf{_bY@c`y8| zgXnJr?;t_^vPp>g0YFgzk-xilxqg3#<3J1)3&n#wn|b$e@n5u+|K2wKmd=VgB2Z8y z!W+cml&oy*e}{ED)ctFC2Sb8H_F#Vc?GjPgzp(HO8V||?I*z}ZnQdSwl9;CLa^!~z z{civ8+pl;p z9TkZXxc~+1ok67A4=LM=`~FfCoU-wtE4I?h=4*2dxAwNaPrn=pnE> z%d^_1Mat%HuM>ua|K$>iq45sh?d^{f``MAAQAn@tLIfI%-|_Xka@)=Wi1>)pi2x1+ z{EvWo7k(T3o5m9vv@K95^2b*M6cXVI0-ny>yYKr(bA-aSBmeE+1pTMOqW@>wAH@H^ zf9Uo%AJR;_ZzM_j+PPu`t@?>4Q&}!Xvv|{`u3`Jwfeq} z4lP_idco^<8(cg4?7mRFz6IHn!NI}7W1L5-w*1$xZY;d+9+GJ@e#UJjb20350Y`R zuv>l77mJ>**PgJZX<2&V&%?*mTDCZQFKs1jV74ZMd@D$&@ua_=bN$d|$I8_U`4?8W z@;e*rzqE%dq~|tG_}wmJvk^b4n^eps`|8Ai^@Ny~UT>U>_t#7Nli#VEEIEOjV%&}o zjN$CDHC72L3#}X;CW|6q&)Fl%N4S>9e0x>OPCkxW>xIhJ5@J7Iib{b7H8-mwjuCFP9rP6Wy~fST5GHn-BY& z!n1D;J^na2)@P`_;XYh(B=aWx)geR0I*63;!FYeQ5M9ms$WVxNS75dn<;PycvDwwz zhwCyeByORt(^{gg=D9u-)!m%``ssSb^{XmOF_Q7xw$9bkb)WO=V##4g3r}Uk!=|`- z-}S*|m0uQ_IrK*`@GI6G)o@?9lG|n-XQ7Kqv^t$I&y>!vN;e>KRN6pCRGJHuZ~ziA z)GL|GZp4&5r2YBM3=N{+(EQnKo(G@H1sfS-h2bR=yRci6$S>S$2*=wu40tFs87g^V zSYm_{s(T$a^>93>&H4kQH*T-R=$PoU5qK6QVw1HKLSCe9-O0B~>Q2^@%1`PEeCPiP zmK={9(`(af=auJIJHj{@_U-t(%hXys`G3igiNe zQ8NGtOW+jre{oZi9ov4uM%P1?=4B~a%N*Fvm2IbUp*&Bk)Z5{HVb5f0zkl&Nh=W(# zM_y~jnLAjG)WS=vu}M|oT%twVQ=wKu6@tA6?$(qT(cht#g zWj$13B7tE=-eiaQM8x&#U?M%TEgxzO;I- zoAy-(UhP2{Qd^#kV{dUOVJm~+QdlZ2erc%vegNZ zLY-6kPn(qnG(4j(d`O+>ydZh6yW40BL+YDyv@|A+Ip(McSP7+r@wbpygA5a9E+4`f zoqGlX8MH+gk8rSf-}4QlEqQpj;E2z)_Z!z#=E{8+bMKFBRhM7gvK^&(_59GiOZ2{w zxec?qqCOLu@m2O`=+7S#A6lF;&$g%_%vX16G^oPgJQ(yzoTc)T$w+c2YH#F8V&V6B zUk}(G>v`C74_;(`?M76rOE~8X1?QGCF9inrDys@xu-OhmuPb{ZI-sK`rOxweKAl>u zzIuG~K+Q0UjcRDyiahKX8P!nM24E8_4HDcF<5@(+ToNC%aFz60 zOP$EENO{Sf$EMsCcjs7G!G6?K!7IK`kaesb z1-)k#j>Zo}1r`$akC@x2UY);i>!z*`T%md6_*ql{i-J*+Z8N!zvq+7}5a=H?t5R_B zj$D7s2DZUe1`Y~bDmubechD-Ed zkgq|zvfB`*{l!aG=k(62ob6}xPEx*lwWfBZ@rJmtt3d3#i3`0CZ(;kS`-YAAGAm!} zU2M;uaC8`=^J-kHfOihuUm4!4IV{n_l)TDO@3-jKxkt6CxZ$d0q>5n@?aFe?%ak`2 z^D6UVMK4fH`7*Jq601e#MVpW9wi?VHs@B{VoBmiGJ+d1PZLWW9?b3WuMLeg?o*H%dJx`YdEi`nq2sJs!-*4X-hx+6 zC9>`;Tz9?#Rt*LrY%C0OE-{`6DDyt`VlTfW4zS&r`_MkN<>9>CjV_AZ^ceJ6Ul}O6 zw&gLn{&ef}N?pxb(aoZ33=CEkY@tJre8oj)aPI87SB5Y&);Abb<0ppNhmtK`D#1J{ z@=qR~=&aRV5?QZPW_)v}qBiw&pTuqU^H)EHrdlU>RbdMcv)jLX`q|oS&Ufyz_!#Gw zVC9BlWcY2#EjPZ!F?`E&^omG((waWQ>D&Gs+M6j^RdGco@*h(vzkD{q9Js9JAp2@d zVoc8S#%eGBeOt4IMSakR!M5_|)`hhTW1Ds67fvX9Yz;5MRpQT_x)=CS!m6x|hH#>9 z#pbj4!P-mIq|YG}gtuX_(y7M2nUl3wOLDD`s^@gnQRIwM-c)oynNq8Ex_zHS{?&N% zNu8P|!oIeg9z+}B3!cTtH}zE5jT*by+ph0a}PG0%Bi(Bg?xPLf_Snnc<9OL z*g&C1w4~#g=MgD$o%I#opdaSEPZR%~eIeCAa9c;gyEcEvG9V0eq$Cs4@>5ri%cl!u z%B|0a>q(_wNuEC!UK9F6t=Iz>$uS;~kfh8nX7^NoAIW^wm8`S(NMl9zT^u|%d!^OF z(Z((SJI4+@#ZBL|AvCsS1Q~U9c*4mx-T3qM66}oR@7g9{CeMs|JJ*XV)^2V`g-$al~)pJ+D1{c!%JbXx}!yDCvL6MU%qwewr}UC zcHnH)wGYJ)ORkyZkWC5s-=TLQaAC4_bLd}gHgVP2%tD!#O+`EvQjJX%@%Oo;jGg=s zOJb)B$+)QomSuwl#ZH481)LGir9ll>1NW~VE|IOe?|+H>aj>o1hE~~>*MpqK5P@U5 z$yZ+;KeN|qW45o)3xl+qmK)_eo|2h;gt}!5Ja%>Da0bV+O%16eTsm!Ooc>HaD)rUu z^<$F41f5c(`k=;s!S{4ul*|+*AXc^Q@t)y@zA|@X8k+AcA609p9~rLP>UtTaDX9!NhM;KC;(F3nKq!f8FtI>J}#qj!66u>e`4{^5)2 znW_>|l-5GGFXt9^2^z*Zb~4fmlZRc@>y<3or(kGK$>N-1DBQ)$-B;F9*p;80N2hjI z)qtuz88!lKX31;dzM@gAiV9jC*u*(U2^GpUZ|i%@eD&flOe zn!e>AGbSaR6ey9J*T7WLDo48Pwx{`w#M`xxE@XTwEz1`kGi|@)`oUNn3YCT}RKKZerDepC%yI|mc0x7qqjTWWRMSUL4ph!mUeBQvkfOI5d zRzjH07qgLQ&tR>r^PYl%4}D0v7i_hft1s&FEL@3X^s3;U7!62AUiEC6I&rt~t?o$4 z2&9|;6DP?H&H;uj{Zn5-r?HFP3v9Vuq|!3`?~xUag{?DKRMYGar4X_YxTod`^o$-2 zc{1iCK`BcgfhXg+M#}wy0H`su;-xrqdoM( z_5nQ9dWsa;#75%%|@cgs|TCgdhe7{!_uoF8{%!z~$G)}&GepZ>|#hK}=kzR&d3E?sRS%+9=DjP5?cQ!rH4fbc@kR513R zqqNTwm=`UvcWS$J`W#9B>1PtLtj=pL7hRyz8VxC$QlDn9dA?e2m(+*JxnmUt)MDSK zF0!|Go%RWGoN{{N*G)?&pt&bxFOj>dQqs4~70_24i@(~Jj#phcYiM%LS&h0SZgr`r zEQrR1g~H`Hryg3tozc*Yl;6}WfQpoQ5A$9@!Z0M@Qhr!=jk;N*)Oc-PegG2>=?>D2 z{5?H)Y|PC*SeM+2d%qZYuX;%a zeS~w&SKN2>nT1c)dts(CO^go9$hLH)6^LW!)AvJ(N2BvaG><*-&|2Rd z9}RvP9E%PhRfxL7W1hCkS}BY*o5-i3teg?W^ciGCxZ^6k<2&3+dG15^M=P0#7$osY z>mM}%WXxQ}u0-VM-T%<^AZ4@3czAU7jz{Z}t*y4F$1w*%wMtXJ#ic}@7Ly#xc7aU2 z(=8WK*PnENd09_hV%M( zi!#LQIBmYF$qZ6}td#t7@RQ)y5M$#Pm7zz1Qr-=^YvO5L-UbZHd848rb|+nCUtB7q zwMYnUPPui@-ubmz*kv8cSGd*JGb1T_{kX;{BO2b+*_U=j%BdDN)y|#w41R9##j4!I-*P=KmlVzN8m9ui_eOVIr?E5lYL9lULg$7n-qNU5NRyJKT`)y~H zf&bLlGL=up6*Bg{-oBsO<(Kmv8A^DiPTqHq2;lZQ8~ny1mXCp|y0j!pA?@*;<`tGO zpK1#K3f9*rwvZ!D@()ck1d1p!V-Mo9x>{zZWuz~9G_>4#=2I$jJn2HWzpX2-u+Y6R zq-?YE(y=TPf)D#dMKd~jOkJUh<11@gzNb{KipV29IEMY8ECr94CymHu^;HM;OkYb1 zwLO>i?xpIAoTt!M%{&~mD65`%c-1W}YD`^#dKI#0#oX%Tb1z)o>$;(E3gyiMrDqvm zn`Eiy4rT}jybWb7c@wagO%wTnbI%}KyCoo<6Y1pbR1~ricxm^x~kOekfysW?H zaxMjV=brtVN7{}9`&X`ziW(F4khr(0@OyCuPM?za+9bZ77{y=mTJIGt{lkKQr7YM+ zVw%Ry3;SO`uubm>bn9}UI}_X6V+`i_O>av6mT~o4LO&3m2OeipsHP>Y8QfrEH zI{V%*S=TM&VSPpW=ObLp*CIEIJ(zC5m(%vPS52_gmf2e~9x9fn)9bFe5!rRGa$o;a zXQeA=gkdB~;5Eg}DI=AzHKXU(=EII_8mg=vojK)qnd3Q-_GRUfBO>K)pX_MG*S1kr z>*zq?{x_8lmE)67-`AZdMZU{!JSC=ow6ZJkiuolGLV1I${Oxn?D-pWxtwZE@Wdj>Y#s%$Ju2kVOo{yRtV65zBlqW2|GizNzES@7 z3Zfpv=b%AURmP_H@f~>dymX&?R{jx^s?w_ z7r|DM&wkhrJv7&m5Zcd00}Xa;wC>C&#&3)zsmEV4Yp2WkR2xZ_6U%Mi_M9w}PGPM& zJss6A&veM5;58%G{VVB%)5fsn73LF4LvaZLlFfTUsmk_q9|?+j)!nJ9lWP=vZiCeV z_KN&cWGo`z}Lrw8Qo2a=Z749nAoAp37qWH^d0o{JB56>KeO zoEFb;3CWla$Zp36Dnq7vjRig-E}mgtD|=7RuX-avtRpwbl&Z1g_N_U@0nTpex2Vz1 zo9uR<_-8{{liySoHgR{!l8^SC5Nbo~Ng-@ZoQ;YV=@5&=P+~AQyp#&XDyi_CsfOBL=Q5UL>-yqG=;#o0W(s<|J&uZXE?Mnizw=CkL%h>Sa%2dDD6Vwq#` z+O$;Gmjykllk|8VF|a)ArODBhMOke;HT9#nvVIU3#H5baydC<!cRJ^-xR#bNE<+Zi)6Q*McDp8Z44lE@D#aR#&E~!ec@| zHjZ`qNrDyHe4dz_SNB&?wQqhI$*9Fthsz&cza+juaWl!|$pgj;I@;c+_Cx1sU$9i8 zm`YKMcRZT%QXc0Rgz9)63D-I-dwe+}T$}odOUXH=ND1iRJx-HL(FSs~LoeHHuaZMH zoarp?ncGxfEl0n+O7#ZDzaGEmR@3qpj{Ym`Lq>i759$HzzS`IDUyC` zL0jv6iWwK6`w|1WothZ0f{}x&ResukC(5C%_r8L=p>Fk@jmpC;R%Km7e2mu01&3HW zq`3y1Mb~z;q@}jH)sD%A)#6T$%u`A_X_Zb7$-A<7DtyWhwc{MdI`lsYjwm)o@@Byd zpY4Cb{~UK2xhmZAxSMtG%G=~wHtq~+m+`1*jW5zg2QDY2v`NGzTpVHWlH|M1W|XdQ z`{^T*7Rh5$1=Foksq6>%vL!S*hJ@fWx5E==A8(YU)A%!mjvi{6Y>DcQYdQ7Q7tx7P zX}3l16)Fr7pTY{)3+op=pEnhvTk85!d;$L{xO`uFlB4(Zgj3wm*W*j|YH1&+@}r;M z>h*#?NDrehne1R%6M^u4ve+XVs(iXjV!oL6Fr{BYt?G@V>t2fM6n41+=6jdU?N$_YA0{63Y#X{7d8%waqcGfa2&|lU3 zIB!VlwX#S3WlcnsCs~T-OFB;8MbxS0YEq@4Lkp4JxN|no;vc|He46wAN>2MB1HbX1 zMP&qZqAPl7(>;BPM4vA3{iE56@oPH$^RYTvYSe5Q$S##w{7N_9#}y0OOIOKwM+2H+ zs?GOFn-l7r-UKx9t2LRBMLLnn@-R3aZVp+R3k-Kndvx+}aYze5PReX#KQ5u_0JN>v z*$Z8Fe51v4nJXpRb5BB+(=ZPCiC(js&*M7o2 z{?c`y>RApjTQZAOtmO77hbUlVJNgYDT;wq~KbjTL zfrKS57us~s3Zt5t_pMr9)F_p`ugeQ-cT#??cY(o_TyRn+W2&8$#@fiwZEft(=i^;% z5mk>5I^nEZ8Yf62yM~2bV-XT;NPb?89L)~L2qh&puKF8$z-rra^;zkGgT{(_mKu=g z?E2Hl9dDE-q+J|ciLyzvQI?cHbxS72qXlREjA4Q-aVoU>w8)@4g7lSm!&{!bT+ZQh zo1D@bm#Nyj?A6~&3EwbUway(iR=&yZjVS7(C9*L&^L3y$Ni+$OmCjeLd~$i$-lgXu zp;EWfN$o;S1JUMAlDr_>wD@*RZ}6(`Ia%_e z9e#k+|G0y0_ltxfmmHe%N^%*WF1|Os^Y>_Cnxr<;xh?qHsOc$G^Ve9TJL?lc>`rVnKAL0%uRGLp=SS$1r`!N^lJld{&;?YFe{rM4x+pg)h3$d;T)oh!+6<4|Gpw=6WH60VBhw-Q6_}_nmQ*VCF z8%Vv+LVtmS({;Q@pFo%VAzvtg&DvsY!XT|!j$h!hhzU-JMpm)=!}FS)o`h@tsqy9| ziZ%AS@Hh82Wpd_Ao80HMq@0FrkxVz9Tdo}%?V}&}noHCRHRhKE9T289`_Ptx3 zQ`3?$S<8@kdATUx;;W1Wr`nM#`Y{ptiM)7_CE4^g*&KVpP$+UF>`{Qg3!&4nd7)gJ zayCrO!JJv$*%!L6(rTC26&Y`;Pq@btz*06{4uuL~yHW70QnPUuziz3~Y~z_ZP6BTu zr&VXjj7`wBY53;ZD=m9p*&YvI*UehaHUKREFSAw$!ejg394Eioy?d)fGq zW3RAU(xejRB7>-{{CcRqe~C?6t7XyrAbhN)x+qjM{I1N9Q-tbXX{9y~NfF6B6?ns| zm*3nkj5n&*hHDUac|>%+Ej}_}=q~05@dK>0KhohY4uBH#2 zJIr0{lNMhzf8Z<7?Y;a`|JFF-);@j>ExJ=aBgZzDQ=jzuHc)us-jdEIF1A@-iO*p@ zYViijTSCvilLRgslJ*XMO%rKIYIhdr>S#YoyT>Irmpy2*m%A@^pM6Y&m-?|KN%dn= z#X4n@8ciW*9Zxxf6rbSXXC3o&GG__i#^Kui@Phl(wQrr~DaOP2OiPY*vOHsvlXKi^ zq_I~JgD86=;se#0bWJUo)elXN_S5f_vS0XyiC-yYe#sL737%9otf+zYm<&DGZ#7g6@H&DEJO&6jhO zUUC@%R&Pcnyy@?_Rvz#a39glms0%0(j=cZwEdHhQ#M|9I*bXBqx~@&(lV+QQ|hm z&b4L_J&108)#Nyx>}zKw`v~uI{iqBxn6RdN!YQYMfkE@iS|&on{tiFwmE&RU;JD=0 zUamXI9D36?f`%VT2T1frRq)F$tKT{n-6>ug6d02W>1XFYxUV&^noq{@fSfXQ3RZdryz1Jc2>T@QH#S(`8eh2ERgc-^4 zjvT#Zo^xTg=H1FU$D0SJzgM26tZoR6j**#>hM67^^u|4UtZ!v|cy|i(FXya4H%SP^<#sqcBer=cq9k|U8{RJ_8OVVbry>dWk`h)5xmpBhLv(rmif12{n z3b_jEsY*#FcG1%Dn=0R2WRLG83`71ZZ)}GjTq?aLW^15!n_P&`WU_NIkt8^knoK5@ zKBpq>0cBo32~)oCqd*dJQtCZ%@^h@JYG#c+wwd|k?Cy4^W{;?Lq!O~%(2R7tukeoW z`NDI?x*lFl;dXZgH1{<&Qklz|guPj%BfkP(WK0eoEPn`#?U>&TSNi0K+4sem;e5Z* zbU@+SRKA@6zAyL;bJffOJ)4^OhFF)`n3tOWdsfA@ScZfNYq7aSx}YBJS2#`WjBnXy zcvR5}UK4F`+ScKe6k?x-B@!h}S*bn;BTeC{j{aB*o>%5_mU<)5E|>6O$+!NMh9mkd z#Vx?)y5&Ku+N&4ON|R+aO=97)M+=6kUQ0@S7Edy1xEJckP<)v!;AUE&%)3cGxZn`r zjf}5Y%?&Tz($SK=sMmK=olgDCJngG}bmDJTM~9oT;RlrNo=!5g@R`#_CDwoauy?8U zM){Kmvz-0OOetsOI@miGj^JJoiu86a)h5yVd5ef@4P5XaQ21~@V}{HN6NQ|Kt(&gbQa9v2hu(~HpM zw`J)yBQMqOd&X)eI%)<~%VxHivCwHX zUHMmqhFNhs&&Ws0Sl)iUag;adL*<-bZeGpTbGpwp5we!2$6D_@E8-$LgI<(qiwE*Z zp0}r~Aw@>W7MK|2XI3Uzn$}%aOY@3fs1$oayKs;B>V0SD*M}D$cE9t+^U}m@@baI0 zntFDuFnR9u)BNhk=ByK^hlWR`{4!AxR;|vJ{&0UI`Y;O}J}Lw3a#71?gg6|XeaZN2_H@!2l_ z!q~uOTK}ff-3=ep^@{v0o0`pM3j-z68cJc6HJ6x&6F)MVguRp`6W#Lu{JKs21<%cg z1a!G2bw)(rVKZ&M{2Z@WGcyl$TTE3R%^nEzeV%t=NLMaeX=(RlyvFM$nX4D{;%^GS zwp%-e-?$bQMhfp%v>TX>O9aTaE!}_UH7tZn^S*CBpLvSt z^PSApQG4FM*c+AR(E9mJABFaOY%gggxEC-z2q9yYK&%0Oes*2+Eap&{YMT>DNNS1MP=jp?AxSoB}dR*XPwrah*d{% z@B4}fCulV{mdw2t)wwz}W-{?<@8vII0<`0G`Mqpaa)mauqc7AA=ztX1s5`*!T%o?= zuzxGN<}t9@8M5xM|H5%@qRSz=aEgz?pIinme^KaDeRurJ-#<3q|9G)%zv93<)1GD3 zrby>xY{mS$?g%mOSb`=^ySvT6O|L{7M}3>1)vE}88~AiqaS0@_mzZ6*zA-rHB=OYx z!%V&=b61j^Uhm8DW8QNHm-1f&cE>Qz{ZG##;CENsnv$m%F6NUocG5G~oUpIIcBe>x zOQoEAm`Yme=8hA`x}Oi=E&E>YW$f%c=YJ=-X`+42W}n0R>H05}o6LeQKIgMo=2q55 zjYAiw$3^<*4jPVzh~B*NOm()emei*n+f4Q~c6U2zjfU^8w~Ll; zy00T8GwHp9zyHDwmgmf@mQm`G;3hJX3e$XB+^}KV-up#pctAe2cxL_{Jz|~>sNU{H zkusZ_iOjL&3#T)#oHQd(-3<-2Nz7QHZ0D--FTD#pnhn41`S44lw>sX?Iu!#-Wgkx; zGqgX)%9iOBV)INvcKZE%-Jr(UKFgIk^nwfVt=K~n=eOr5M zKBpa=XhPNBR9Usx>G<3uI%X)yxkvd_;NCEYij)?P9a{2E3OzK037>RsoFCXVu(apUHX={fUU6X~Xwv$>{Bs5(gYT}kWR933M z{Yfh>S=)}cB+1z-hst|F*l1G1@9w)K7BBL*NSXYEm8qOL<7FY2G(jlWr^Et&uhx8} zvfMq#37;#TC(@D|oE-~$^tspCDXoax>OACnl%GvW#~V6IHIX5Rd9qGvdl=8shjVYaFAn5e{i$w z5Gj|vx=RG+kXW1ti&lkDAj;^=dr>?}chFat8}}~gSx5uLLH;-Dv}8d==Uy|4NPoc=5?30_I3Nm?w{iwK2>(j`NEa`bFqgS&dF8K zk9eM#SZFRBXkNWf{*H8Zkkwrql&-{S!-P^$B0cQp!y^sXbL^$co*5%4L5I@Ej=)W&vKbD&QT)={F!^(RHq*_zC^%G)^_YlR`#{b?xV1A*evD zEK0I64QDqpWlhOaDd+lij{=BTsY1guXQg}@{VQ6k#&(tS8#5#U_2UIsNe(>_Dwloc za;wBhl>)o>(8-8_#5BT*OM_F@50vz;-CbC}#;U_=>`eB&o&1i7wOqNY06{>$zl9~z zVh)i}q@|WUsgq=sq_5o?f1>KBxkf@v+)?YHDinWUzQRI-oaFvqhS9GVL_&E)XkC53_jQ4f zhZOTT{ehE<>@FvaBu*SP)ys0Y7jGz0dC5>>HG(;os$CMtP?lXyF6ttA%d5iyGE*C8 zp?D{?Y=qk&#pSN-UG9E|qh~bvCoFRuUO9Ov#Sd}JGRjL7g?ZHqOXME4C}hMT(Ix64 z#;-qL78;)^l?!aT94J488*?xA3$J}lm!Q8)MQ485$=z~)dhWcwfal0a|L`#BJ6=Er zS}BMQPH)PwW?@e&j+=O2uP)g1{A*8+-p3cIE(wQB5zo+?74K4Bl(BZulZcY4aWP*k z-aRu7TU;-G%e%1k z?Cc?`XjSafYHu=lvvcbNZ!FV!?m%K(g_uA2_(;rMQ{VS*0~T(Nha!ev-mqqb0!o}F z=|lTI>YBJBu<{?S6kvP@Rpi$ihNbwl6Yg2#k_kF}SVZICNsITXPZ&&1&zS955)QH4 z*X~W?I&y|q!`J%Swb_sEO3H;}5EvIXSBtMYc;LLP!lW(jRZS2<|L3O`RCQqr zi}PLD%nS5EQJ!MJ#OO0hFS%l!iP0_zo&}GAcl44vZ2HS+CCfw|{;MkNyd=B2<)*1vrz@OJ{M zQ|O(*is#hVB?Xhb#ghEW`Z(o7%K?Z-K2GY+l;Nx?Se~Bv=H;^X>)x-=y*`xQ)$Brd zEh>EJ@?)ErL#1`O&C^u;>NaN*DG6q|5*rrQ)rp&TgVGP}UhV5bWveu3=AHHl3N^gJ zxYDSZhf&{&u6RiG0;#9_3I)^&`4>AXHDPA|gX0S$h2J^8*heSv>+ywbX!dDyg#O7# zPf18$A3j-FsH;gzFKKxrr^eO%)9{VGW({h!XJ2*Qy;{)Id~PXeXiaZ=^GF`Gp?IR| zM|V{sgfh&swDDh1c6lv2&WsxX8kOC$Q={Bo)&( zl&0j0dkb|^ba+b00qV6Ts|W;-)#jSRM;FX4 z+E+}TTx*$O$=2TWfxnx!rs}Zp4WreEQ66f|e4RR7dJ6(pwLZz4wy2ib@j( z?%^PMOWQdurxgzy@c!Cc4u7{tM?8O?3+4j6u1qaQmx{F7a$#zYlRy3Wiacj}INMj4 zl=%et?NGy~t z#cwCJMbS9BOG6w|$w0i_#6KG^HEs!juZJ_!yV4)R4Q=$6=m=eCCdjvgXOb^e<+{K& zTGjLPZuF*dtso!5=?bQDVGvLXDhHL={I}Y%{-yo|ZU-RNm5ykCvxhX!RcJV@V;%9< zmS65411JAw=+59A!s)U!;eN+X<#J;%`18BS)<6bVdQrhXad9e$Jv{w)kVs$o;>~bh zzqr`sM<^l!9-_;LxY(V4RZJlv=gM_rf`VEDXN$qSH39z%xcb|C;{S4)MKFQ8F-P^6 zpIMlwu_LV9Tg;^C60fRxTZ-&in9c?@^AI1=4fM&Gf9+*vAt8^(J5%!sT$mG*GD>-s zTEHKOA>gN2ya`NRQl`E}rL4kDbQw_w^)Dz`Mv33K=Bxtt*Lsi3_>{nP9%fa(MuA4_ zfryUFzQEc}jV^aa;Jx#D!km0Uj6&}N4pXs}f2WNf43CSRjyDpkQ-=_m;f>G2CY()3 zxZ9SIv{t5E3$PGZiH|Yzn#LEhIvyQ_O7%J33VlsB>!sV!=T%PJIq{R`5tq`tX5BUi z9H=F8T&(L|oWSuZPB#JTwj+VbZRvP1*Nr4?N?+0PR+(^SD+%iRaD8&-#atD~z-@VV zZ7A)>ZyzeB#3ND(1#t`OHB)E*ZboxKS0t8R4W&Dj+f@-8KPJ3YUE{Cd6R2?v%}C01 zsL!`qS(&D#DHG_EVl1xnnkXC*I~0#8-52!$W_JF#ygR7A+Uow=y}ez|_Ic*(&dSnG z7hPAXK!PD!m&6c7>mM=1(10Oo52KGfm!vGfxNg#ZGj_3`(kbm>on)1xxw~J~=mJkQ zr8&LX0fM|Fe9Sg(zA3Guv(Dwj`MKDl?TtBk_2^F+O~octbs1w%bZgC6!2R9OFEuVA zvNK^;$hFYLLk2f`1p%uACTTiP0I#nQ7XcUxGIkcIZyaw+@BODre=r183UYuTXfQO{ zcL9Pw2GasA3UInY;Rm7MlhcW0Y~;~|jkeBw!ctN=P5yj94dQ_Ysxx?jwAE$~CpqGB zsSpcwV&p)SfV_z41OJG^qTTJ~?>))EGCFlNjAqLTylRcbyq(c%<-WOzR!6*?>VHG{9bvbf%F^wI>KVg}%2=rvwx_ zQ73qmb6A~ho%JEDU4Rk5sTa$^ds*#+&tzw^KZ{cvHJEjJU2FN%>-LB*?&fc7QB8Rz z#)8*-;yG7~@Y7J`W>PChIKJVaqu2FCAEK&h?&b63%G*b&Otsfx&cqlD!Gnb#A+xcN z-*~L~f)2w2L3OR~$3-JxF(J;72G%-Ev||T+S0&kgGbq{l!C=ayc^TtKz*DPc%>6Cn z71XW@qqhb`Q%s*TQpr^QJdCu7>Nl31o5frDG2vj@e|GOeP|v}3gbc@C`>muuV+{5V zvi(eS2xz6+g=?4e8W*%Hu*VhQ8O{(r=okMp+;3>F?*@F^IlW&6XiVLsnl|&^-uhCEqR$yBzs6E2X1Gyh!y!y={gZ^`aD%)ceOE?Gu>KXEm1r$02`09ywh?rctxBki~Z zSK#-?RvlcT(+18oOVKb&g%nIn zuM!+hJ>IVQjZ5%OJ0P`K4Cn_GtebRF(m9{i7dP}J@meNKWx*rUz}fm%j&tIwgn7T; zin4I$#w}v~AV4p6@c&y+uaOfDyf1_0nv}t8^3(yN^as%*T|qH;>}2gFdn`i@^ks=jZVMR}QHC*;Hvk z`ER$yUquwIXFmU}sh5%ZZ%w_{PClKU4z8aeU;n3)AS*1P+i;l|{s)gdD$VXL0Hu>- z?f1^4m(A;!gbPFTl}`I#l$(?JK}_vkBb%hg(FP0{)&!!g?Amu#_?Xf3hjzGu|EMc@ z{qZkd$w1`Uf6|rcjdPs-lYLVti`y z^?1k5hH*9&$xy#(L<0Z3l?zkGU*GudIi88;Uc9e0-dM0vF&E2=LD`!EBnNrJn2Yd^ zieNB*Ezrr1cpQs0)Db49yAL?vgIuZzm|o`JZ{yQ7_g0F#oxua=LMq#pT=i?p=&)K3EFV+_%6~D z^Z7p993tJT)weXkeDE~_eLTZ_(A>ChXJ&&-8JDICp#w}^RcRfrVvEi`F5mCd4vX|u zu#4iek&5Dnm2WG9O83JKOE$ zYL-60SNbl-;*-YUkI|jOA$`|vY0U-4N>BS1pZHW)=?Ip1f9YvU}co5$iOIn zPR(h9);BN}R4LEkvT(BjZv_>ge+jx&zzWpbtIB%*TA|K^+1OvejgJH0o%J85K>7~v zC08!ocM*VE^sD0htJQ!Y=rA_AH8$!!aDVYFV!nL->1ygx3;f|_b8vV*=BrgsmhbcV z?yS+63g=gbl>u)X5+SS9X#@SNpfOyEc_7w@aib$+gvb&D0)^z{-FnOU>DBoKqM&b- zxq=od4Fm~b_Q6T+AMCF7{Mp{=&j6=ix-!|;UCVzz7+5+fTC#}ub@f!wy)^nMIO%C4 zc95Rri~z3eThqG9WZ2p+YSxi$&Tlv zu>(H1FKj+EBzT1AF&!nh9FriP9>UjXG0SJf6oNY|YZ7#1{#lhZ;&~pM+Nd5q&1t8< z`GwZ;&?|zo&*qw zpna&|2*LdvSeQgxvLt;tyx(nI{OREJbLOl6YS|vwSyB-iuso$<4f(>C=OiwhBofD= zZf|7Q$#v*b^)mj@aP=X|yKY6st6!x(krdJDfR139AfLVD4aFYA@JVdbC?3ARP}3|L&FH7GCsIL*RJ}Z~JY}^NaIh z$$%V?c3^g6gC(+|6p} z%Ltu~f~-@RS7aBz(CEBi|6CRUXVl0>E+-@SqT1#c|lDn zgUq@0K^B7lDds_n{HK^F|JS%Q27ipU91dyZw!g(ZnPbB~=G;_Ll=&b-fIN&xfPkDT z0@znD|t8Dwj0%l+q^#v+8kNcmPm-Vkdmi`#>Pt3aqoe#?U{}=OeQ%Rjj{&LAF zluFuAsP>niJWQ;lX_o(sydjkoWll<2C8x}g<$qqv`fI(jD!xT9e_r{Y2_W%i|0qoj zGv~(PMIQ%7k*)TfrvDN=0l`ZmiciVMCcUT^)C%sdsMILYXWVDE#!Qf#UnR?)9JFt- ztNN5DXnfL%PomIDBb+^T6;;MV&N^!@-)4w*v-R+mh6B1AcAT_bE$e?9@yZrlyJyi7 zf0GbjJzT>3&QL`;?860l**Wquouan;61a-s?VvM8-65Vi`6>t#VQDdx*nOFh?HFNj z+BnPY+#P~pGvAKHXi!O3Wofxtzl0Rex(}Y2zVX@tY-)2*A#~VS7&W4?t=O>sAleo@ zE{Kn59yn{mHO7s&@tt&UV1)ICQASO}G@Wq*%_QZLalFVlsW?%=MVIv#yY){EVp>5O zg!bXFA3xJTNUa*iG4zQyWQx?K!tad5ol{_rvR6H0pqkQm-n|#M7hfhfQ_`h1ZB1=? zIL1$C#ve{bPcOi^QyX^V0w6wo++2O23{D~H5lkn&BIuzF(E2LCg6PjI43ZB@E91T& zUfy8|Sx0ex+kpKQD}pVZdKO9{?1q*3@qLBNQ5>HD0qdi#EH4&1C%SxTFu;Om{*H8x zl|@k^fgqnEAtv&8Cac>6kiMpmRTV~$&V#Fzr9lT=SQ=b`M#p@O^bbmB4bnU}TxJIZ zWBP|7?|pW^Kcb7xca$Rq3!Yg}(t!-JdvSHqGon5dS+>pYIZQhi#3feBovAhP7z*jJUG-nLGr+ zDX?HzTO>jraJq096fTjVvKWK{5YH;1Fb!$~!)h7V&HqjBL9b{?ujiW@=`uXt+P01h z>v8K7j>8`~R*Sa03THdR5?-;%6D}NxLeIaNQ;u?SNh|XEsvZLr`@3ixP~5OztPMyk zO<>G0P>f7KbTK&2G_bYsj|uiz#3NHM^kq)}2zGtM1UztH^1Pk_BfkOu)H~l^McT=s zC}D6uR;lIvx3xN;qV;y_7d4YAxv8|1i++%}T%VFRJnO}vUIejPcRTImk{u%z$3Th? zWW)4+2zpa22b{s7lQ1We+tsk9Sg-qrK~AFm-#KMJQwsie3qioFQ3|qXjLxD(BqsVY z?NFpu?tj5=414q*=X1IS_bY$!n+Ns3@LT?{phx4`tkD@}QghBNgna@bj#dU7NLdI3 z0B+c6wirzKKcY8;Mf3z34nTrnU`R)bRyqGmxm>2&pA7nvFw`eml zKj^9=HAk5w@ba^nP!ZLoljJ;bN`b% zEK=Ltn+SBu411J8kbux8U7A==FfPBmY<6lE^_})C?L*vVHNMXQvepm=tL$yBy4CF3EIxJYm5M-Z_j`f+K(La&?8+ zb+@Fsc(DnVHNRw4!LTs5Z5vj0&z}(KX8x@G&6f5;k4~ZF?Qr^pHhFm>oL0&Np?u5i zoSW20U5sR5Gf!Q*S=sLuW?0K8Y`?zgP9>t@)xAFzBy*a!t%uoZkmikUMn%FVB{ISE zR0!r2!nKjwuSM^xLX)82a*H4G>jT+42e?1W2)Q0H+4^5Yy=a%rkcEzw~(dLyXNCRC8p=X zMfMnDw5=90lbOV=aWq%74n7P}BAshor(D>iDM+jYwn{}IxPJVPh>X=KPRF6&IVl-k zq;-9mG%HxmH6Xd>vnllQ5;do((_8BzBTT2g0oy;yWbdrS`iX-=8@Cv8Y0+D^NL83q z9ZcUoMM+Th%CL>57OI0!lP0G9^ z2%~N+`&g3@qA|af@q$fu&5eWAACJ{9-GC!ZZ>DncwZ()Vxw0r%}A+=EJz(K3*^nNX99?| zNe|4eP1gmU5SMxHE*E3tlsAsTi&=;3bgLqmM0_djQdLe#7NkR)BRMI0PoHd8g!fCe zBmm~2!YwX!O&?wo7hi9;M=_#P_HChPC?!V3?sWC*S?Y1~=f`dLwuSDYXmcNx5zHY` zw>ONq7}x47kuFudbljl|nv>CpWHf9|ZJ8&|7DZE8v8YnMxRk2Pm$6kyr72S(3C3&` zy{%{gPH_rkB7A1Ezt0TFM*C>F_M3e8_BBrVuKbd*O))%u4pJY+-f7XF+^1KWVk&vb zUY)2-bfA4xkK%c-m#zor6r0K|Djg-ECG66^#;-oT$uSj67X=8F-uQSll{U&ggQDw( zo9*4)+@3etwp+e@&OSZe?i)X@(yzMS9g=DU3H*$>Rt^lZ11Md z1k`_@bh!Ic{<`F=Xz%-Pm&3Yw4oL)~=tZU$fd{4D$=zwQqwHQ)79|LDA2R4u!OO(J zzu9%2;H?oRUAl4h3^m(g>AJ1PFn@3Qc}+LrZ2W4}Xfa*`zFKEj#{UPb3YPr z($w9unpQu8vGqB>A!oKx?iF5<$ee!C*$y2lcp5XGbq|5tisxpVW>h}|ji-09l80Cx z#GpPsom`o}9Og6(3*kmr=a8sIGtuh#@*8}Ex`yRL=+}D&)^3ZPY138rV|Sa$OW{p3 zM^zbbhqVg=MkuR1GUL zVo)dN#8~9*?tz5R6}sT_F-7gGy9*Wn^Pa5EwHGOljN9mB>#X}N#tZ<3N1bi%=6Kat zliZ;C0cT0Q0sSG&=t+Wc!f(Mz2}s&vEoTat%nN?izFUD*RS9gncy66(H;gjTaY=luz6<;S0=|I&UpueYVk`pIpDQ>m4>NynyI zzy3}#Dp`02wKym_xhAvKV_k00c~Z!C_J9%bDO}Gi{<}A=nu2oe=VEK9OFhQp-0MpT z;KE^|ph?Ol{mxg#S&cI+(z2+5hnlfiH>JLtpN=!{Fmlbf)I_&@8ZEVd4x0PcwiWK3 z1~FeD&5?(rrXqYhQB&MGwvJ&O4yF4-Aq}S0^cO7&R}R?Ks{j7u^S#)KngLxIn7F&F zc+CW%p*zfXd&8#kv9snf6^PrVLHHE4c_jn2F|&k9-purr8eUUYcV2y6Yg}=LI_Hp? zeCK6Tm95AY)!x<->P_p$D;n^cHGZ?v^p728va6FYldDod{feDiWkp+w85&T7rn52A z+f9ZO3O%@!MJ8DudQ2c1!Ot#i*1s;1ed~kHWz}MGu3mDHDZU{^X8TTLX?ez3Xu84! zDDDLB{@Jkcst7`Cor_xKuE5&u5_fn=$91)ST=`|ELz0wZCHG#?BT zf`c#-=H-GueZiX5aoH}XaPCDCtkumM;_g8&|PU)dfiwOtAXPPI8+{oj0bV!3zx{2a~yl@gda^GcCA`0sV8!nEbrk+Mcm7sXF4WfIJ;sG+={c<@Q0Zuq}3+)3D zl3q^jhH|ZqEXB4X(CI%3{jO5MT<#;hKQV{$si8S0pGL)UCw+a~cE}GN?X)Fl0;uyLr6ep33g)-vCvDvYMWweG6&qg2SSTpJAWpFM2x#Gd<%i&| z{ak$R0ZI2*TZ!1yiK8RfBRz|oH%6R}nJVc9(I4hZ5v>W-|K1B1i*Swr$_3%_El9sO zj}GBn_`brE9I7}WTrtMmWvy4kTYaz9Rp@4sQvtk1+hfYpB*|t=LwD{yIgYK9Ei}}o z*c5+1%<|0ewuy%0zJqDZFu;0NS7yBBmWge?-1$VCZN)=QWQn&9xa4ijFCQbj@sJ7#)M`$gTeg|i@qK`*b8*)a@X zWn>c3qGIl4|F~4!disZb)U)kJp>O4nElvCKGw(32LJih)PuGH z4ORl{31pqgApP%> z5H2|~_$$AxF?!(waK#{8TzV&faOH;M8t5GCbYE8U_UzK)o?FdJn-DCDztaVat97{A zkDUtNGsXpduY_(hrNvN+Ykni=bO1bi-_pe~J$3YK=&7<@BB7xWdKc=_-5?J?F+S@K zA$6r_YzeP>rV*&vvT=&nzQ0IcoYJ$c!*ypdq|e#mzDYZ@L3(|owc9j3ad(uaY7Ie- zv02v1xeXZ2TK^dU(*iHfT0j0lT~UJZK2f;3NN%OFS+UeIYKGU@(C25#!uD}II9BhU zAZjJ%H#g2j$NE)o?C_f~QngP1VQ%w{;-?5w*~TQSP-ij7(@#pedh6{)`ugp%dDCJR z^g~zqgW9!V+9no8uG0D>W}@G#X`n`z!LMatL({2268w9``QrBbh;_BwMpjEEl~8#t zVL!^J3FQ*qm+lvZf{Vhb$|UFH!<-czv!L5=A*DneJ33cw}3*H{jied=QW+mci_HCmU zYVdu?eqoopnGdn$YTHRz<4Pu--%Pfr?5sZbQ9c4cB0-4V(K79*vv?+<` z-4-1`*s6O;lP>-UWStLC#y-3-P)keH-CCF8SCLb*8BrSU#p9YzT$Z-|)J z;(NEA?tpHmRLe`?@^7h=;&*eOj}8pXQGpqm4fQ+Icqt`6+YiwFLS^EyiZ)Mrw8fpS{0-dk{#V$U(Z5PJ!MJ=m(gwx_mgOzJv88jjEYC@LR2=3XNvG6MUp&A@{2?gaT-PdrWd&(< zty47cX{{rHsx*}Wi_PApz_Lp#^9{t%9J-eqe=?k)(gOY?xCmmrbUb5;JFnPX#z+7> z@>G%0K?>buA-x!)1a8z-KF}Z4ZHRL)(+e7G3aF3)l(d8gMV|^y3o$RQrzZ~iL zCAn&hl4T9q{4z6Sx>Y#DcA;)z^*dW_c=vVe$R(s!)0mvrVWW6aBlspNlT|oaBiaCB z921L73gdcDcn5jCF&@55W0pJT8?l9ExyoJ)#w4mcCd`AU~a&%HqGMt?G3BS~EiqT1GI9__Z%voQK z>x46hOv&Ns{i^PBSr(^LdVr-)&pB<;(%>e?37?>(4rPL>;+ghE44T@POd{=?*zGeU zsgKj@oLaB5FJP+i_EB!MJg3j=f=PkJcY@_grn`h&yfT_)AR&N85Fx7Ql;fjkBea1JKRy_nrq=fju!G`6`**Rc$kb^} zrpalCgZCFz4pea*NdrCI->kMUi}08+tIUlLcUF_nsq{@&13ei`7B6VwG-p&=#7w=c z(kx}Y0E2C)T5)P2GG@qjAhz0Bu%BIZmc{GX^?r_aFR2Gg->xM)eQ^07sNFO)u8jO9 zPuF?SVki8flAP(j5q^~qseEm~iKwbxMQgZ8s%ec;fNC^q`AY_ri1FRrJhMKmm5x`KIHGG8*95yU$=SAf$O4%q1IqFml=tmT%Wz}3>PhMsiewbIa9vHuNlP-L z`0F9(k-^>5NMLS(0||aADY`N#?suIy406Yv9RHdSm9t9&W{hm9X;ARW5D+^j5;CLqX^Yr*T&8*sn%a1zlGcDR% zAZT4Dt=8kE8+{Y(7?ta`Kt0#l7{#R=4c#ZB#0wr_jWeRe3toPWvpwX6uJ3{uo?N+) zotg4)(QB>?UHuS^SM@=Eto+9h=$Wp5JoACeavQ#hm8JYhXWkLF_+gA`!SiA1d61%RKlR5>@U`K_m}>HN;k<@w+p%I}{X> z7mf_V#C6x`xaKG7qUI+xLCMirxrIg? zxrIbU_?n!mNSd77xrHD!xO&lrAWZ4{rpR(%q>|`=&5gc(l-j^FAvh>aCK<)EZGtrF ziD;UtQ^ISRVHom|y%&cpbq~}rLq8fhvaTtXehf=gqo3Rg!-S^HAhHgeCQxDNug|n( zF-&YE|N1I*J{{9N-yyq(_%t&s-XZ&{zZRTh#>5=0wo~e!F&NMM8QfpvP1pylJ*oWt z+2A%_DZu7G;%b67UX^%@UA#F6Uc4pJ|Cxe@bbFC{&!2e|NTlu=dvnODzKb)FK7<|K zCyzc6;$3T%|2Dpz6Sx2yuKoB1Sbx>g8uXE)Vnlg@?t0vO7@lw+Pme)^N-DnOhLtS$ zbH-$|{+{G4*?oig{V<|Rm375ki+xkq-$&YKiZ3L;eP3wA#+kDwpdM|%O?CI-GZG@X z9W(dyxh;V*U3Xh4>n!GVuC&nw5@51~su!-vM5cG=HDX7d;f!gWoY`;|^ZU!_YBPJi zrLya0JG&cr&s0|eR$}$!VBO=@o3V_B46srssyNr2aQLfNCw3rTM;}_Bq&9)?6BkDt zP3jU%N}&!w%bB2w^owii!Ewr9UkMe`e;!3#hy0k*4^^(h^RGneJUfno!UEipp zipOQiZTvL3vq6mY{P%-`7w?cRr*M@QbNcaTWsbtzyxj>-9aoI)q?h2^!Mr*(kxe4v zm`=e8cJTraIb4HHmd@jVF0^5F*k*H0uheNHC|vEwLze6YlP6=uR>JCa){*27_G&5!!#TXur-B=HaB~$dW^nUgmV(>5ZT>YYl%3WE zpO70B?3yR~ytb8@|J?#HFVxo%I?Jfdqu$q|@ry3|_6(>37>4sNC;_B+q~usA!8eLp zLUY2f*M0e7869xHy1RKjajdfrF|Km#%uX@ z)zx(J@d_W(M5YLA(3?L?Rkde)8?_IXARk^e$HqTbH?@F&_{-(4@!4hzQ# zoU)Az3!0k_U-W7Ve*IW{rlwZuZB8|H%R(eZMMIbZ%B5gIJ$^hmFR$)8&*JBXVnKW@ zr3g355p0xUIn6=k?OhC?xZ4o?_4F9QdJn&HWNA3Gs8p8p$#K#u(ck%=;IPo4wC|9;wY8rxN!|~SinAC~MKe4aXPWb- zQlnquLz;2|NipeUnanVY%LND$d}zlZKm`7~$pJFq{oTmYvF!CqMht|Guh-iR3#WwA z*$_2}Mq*l}8J|!~4j)49K`~@oE_G&3e372mI`thQhmFK<;)BIdy`h1|Y< zpN1e5Nel~Dcj5l!b2Rj%P_c(&(UOV}%>D0Ywrk(OrAO6kVDCAQX(>RVa7UBcV*+H= zoRi2kHIZQ(F-&0wzZWVI@nW4{`b&J73P&~xH#{0ck`qNtU{_3OB1znE_uG8-*E2c% ze9PzhK+vQr*(iQY>G(|#?L|Lg$J$uKhuHXJmUw>RIbwmKPNJSED zRZ4Rf$$OH%fH0p?m=9=NF}o}90`FjtB;|T_gBmhjFE6M-k*3Q1sT|8=5n2PCLUoVg zmObLlR*2rFT-BW%i||E}u{jBM=%c;f+&V792s!X$g90dyNYBtDrby3UR@5T`Vojp( zK-@I0Tj`k>0-duSmb=A_s4nO2T*$7d?pkA0TuOAu{fV1JMHjqFh?S2}Do23#>Bn-` zI?f0bjk2Ry>P_4na!nAF!!H3@Tr;v`Ow$(Zd;kVF-S+}VAUD8}8X}YWP#c&|eH^pj zl_U}&Nv7EV&57W(_V$}%Ud)n67oC)FqT6*?UT?Hv@3%2zD(s;M;Qm2B9TC4`<1e|9 zS68#Qn88~S*xU-9tjgiqdmon{lVh$Xd(c(eHzX3JGb#4p z?0nR|Oml8RhLHw!ZGt+|B7kz=FF$`kF1tQuM)06kaXhPms6O8a`}Xb`0ILT|iMHWW zY7lH3IRnr3bC9B-NzFAE0nsC*Zm#Yz`3B`=VX|}*LLSj>jw(R z*@X(gHx1^9H4W#O=_d$$!Sw;GxJUwV46_9o1T%`8K$=?OpIsOib9xW4?!!vY!4Hc( z&1Ia88;^CQt+l!${isCS7-OT7+#j`T2i+KUlEF=*L!Vh^M(o(uN9>H)EccsIrd0LQTVNataKMlThfk+;AF5I|ZlR-W4sUa*G>?_ib1L!KQ+}PedS9P6IYN zEX=>F+Y9gPFoKgO?V8Z^yOoy1^{CfHg4~McC|;A;b-!@Mu-0}jVtzG4oTbam+vKq& zOduRDZ!OEjlPALhkH__>*Ujz666(w6ZReBLV8yr{mbmB#SF+M$R!yO1r)c%#5qb1O z5vQxl!Z1wc#Y+j7zc;%Dbnn&jwq6(BZ~umhL9ga7ogq?(QDZ$#zsgeW?rsl$?T>H1 zq(9u&w~u|$nR~-qm_aOUxEt`LH;@F-&)8LIf3=VRgR-rg1SvQ;iLy4&VrqDsF-J!a z&Car8am^kX;NWJ) z!~W}G;U?Wqrg4iEW-v;`E5?>%YrcQ`ap_O_szphl*STB4qJoPdSCGZCUtv4qK_>3V zgtQuBk{(Z>0h5bUVHgv;f5|EsFD8BqA@PG?RK_NV*p)SqnF}u~?A|loW1wek$4yn6 zGKJp>BXHf$_9+bhM$ktHgMh_jVlXxsUwk^OLUyaDf*##K)N z5qxSBc&+?z@uIZ!4*qe_4GnS73!?6zwDLP7h!&+w?KghV zPY<@>2=*HR;#r&VRsdvO7YwpJid~aRF}3-v$qjkrlps#(xrmb7jB748L>=NRDq)TI zBnp9I>LfDDfNR0t!Fi&&jRgxrMZRKCro?#S{s2%R{S(#f)7ppn{-Umx41|y|!ojji z{5lK%j>!~+N$3pnfwU7H09J_#=jpFWr%H71&}9dsS6pozDiKi6sL!M81G>;qzOIbV zmtwk@B{97>-)*oj*+Adw0NQ#{SXCskw=y&qwmHhDL?NRoi@W zrf|l5^7>8Q1aczBMIc;TKWHKTu1a{{huq{Q1H8heX?&$dtUgG6H2J~0#6+6e!MB9^ zJMRr%p6rMc;N;9Z_ph7udPj0MASP^R%}6ksp#U@^fr#21k(HqUv>sdV<+L?pRy$fb zUTMCLkNy!8QCcR4w7Bs|ZBlEZ6d>9=Z)K8Qa$Y9PqAh%8lqgAap z$*OI=i}pm*3y7+1z1G}Y2w9J*2OTYg;I7@Dh`mwx<9R!Rx$%g}Va~bYWxuHBsgs#T z6ez^2x}l8PU{X9OUq~FO7n-%<;=r08fjDQ2?X+2;X=910FUaPJhnd&4a;sIY7@LxB zS;aF-yM6ExWt9X9GT2jQxFzHH?WU{w2j(Mz^CKwN+Ib|#0`uWi9Uc;#eH4$ZQE6lvt( z21(Cj827?zh6m-eqspjiBK{QlOb$W>igembfAC7R3H2^w1uJ@{>~RrWYg-!s-PA|d zmahR$oKgd}4BdO2aP^KncAa6&-vLU88i#Ks-<0Pl?i?wc)=1KR-dVtdt`yfXQe#PE z?|kP*fja%bDa{<81^Dv0yXUOGX9Y#_Mo}4qlOOhjmY~8Bieyag`X+dm30MAOYKChj zs=+kmRxycvd?{>$OoQUYh6*=~;8USg0-S^x6Bs$hL=F)=I+O(YSCKbENSGVbfIQKF zEooRW6F+y_i-Fo+|6b}3-t67ov~=9OY?X>p0$%@Ng!F#;2%!oW1R~OhRV+dp*!ES- zS0y_bsg@hmr^IU?x!Xv?VjiZk+&dVD<&9k#m4SJoG*R6xmRQ|%f!T1+Aerrw!%JTm@{#~~CO z(pGZ;f-s)kE7m|H$notr1ya;rSnN#!>V~$8Fm?&M#(RR%UktRDb!}{uIvnJ@t#ph4 z`GfNmd8$!w)oo<1q$1R8h4a3(LyBNG7M3%gJ)S?pNg3n?0x#@T${r)86{kT z7@B>}h^mrz*C8y-`FD7J9anm@p4?>K&=wu&xfjbG_T<+n74oCUd{Q)YFMQ1E`<%2x z+wLdg2+MupMc)^S+HKB;gZW2Z^-9r;A~}$tz0DqI_a@jgkB8VYJ`|x1OAy!zS$n!EFstm!S~tWuy1z%;%x+sauq(g z0L@O6z*?evT2$cMt=8Sm>fdLkIA>ODI|l-U$86Zb+XY+_CSqk+CRxp`gs!50P5~2x zt4tApo9ivUHZJll{n)P>^=;HnbfHrs-`g?-2*5SR7mCtZ9#^qOtzsRmlGP-dU#uuP zf7C}U&U5vF=_r0!S*cm5E+Fg9oUXFtrLhg^C)@JO6sI&vaimy8;FzMP_2k%e>4g$*0UGv8B- z3m(%T9@95$6a=6LFCj~}m>#1gi1oAAtXJt!bbK^9rYRI z!*&X)U3+iB3Ox{iH67^e}c#24It?joR}Kq&=1sgh|Wn^r-u74HHc@bjvSr7Y9$d`z!^naiiPSff*4(&uIaII zjLh6M4h%++xesHttU&;%hGCqkaMOCmZlhYVsw;ljhEgdpFjpubm+ceBBLQ?^brLn1 z?RKRm$s!yz3*7EuCDo=Fa@uiXj-ts<|I}{&#!h$?!(^P|F+0yXyx2iR)3vp@$1zB! zy*t%$(zcu>->1Ja^GgVFu(TN3G=Rxz3YGAQn1W^dt^vlyocx5Hr@QO@ZE(2$?kp}K z5~W~C6iN~I)hrJ`Xp=Y5qoY|gV<-3}pE;~mlH!V6h>&26@BF)26|H#8Hc~ztUv?)CPdBfptBXf#iaE&7 z)hnhMeTc=Yld|8C!T6dHv6^O0}=s{|1b;Sjli44jDDCMF%6-P zRX*|xNe(Uh(ka^a7%(|D-!;#;!tjQR>*RAbo=ta~U*h4Yqm8<3N#W2P#K?SpFMzL6uUgjN3p;E_Kj&yuD>3SJN2GMt7SN8>vK9ch z`CS3r)&fp*^~WHX&IEs`F$}Dze}}%ZgG;$HoBK6J194%qvIAnX{=^a1P#vIDG=g*r1x_PJ^ z*XJ0__q}E@J-mhQ#Kn9fE)Guj&YrYq zHH#xyCYED^3)YgU-*j%X5|nmxo1I?`Rd2D%2^+N0VK6ZDk*w_r(7H5pvJZl|_rU=#zA8 z+w7PfYj?6^+qP|+9ox2T+qP|6uiiP|$EsWR$2G=S|G=!zoSsEz^l=R=T`RyTzBj-K z+qfa_(KdUr{IyvQRkQ;T^Zs{c@|n}lSppH?l|9L5z%~+b8dYVzame1Xma8%{BxCAs z;fw*&oO)&V5+T|Pb}5G;DE2f_b^0YTnIm@Ez;Z~`u)S`fgj?=%q`h3L}xGx6ROk9O;l!pX{2K{L)7)!_ZISyr-9`n!>r{f zKRk78ugGpL5}G%*{zypjj(i&u#m8RD9S2iD@p)ED!wVg2JMkFz7~vmocx^YEHXr9*_kLp z9#(`Q7W?tX5QX+sV*uDUZPKp9^!>2-wxu=b4VZX*YJcx$_*eSEc$6ZFs8FaLd?{r4 zF_iWL4VvW*CZUzsehv#Ofz|XBk>Zdm_Ot8yKXN#u!q8*CYR@CT5jM{FlrguPV;amq)!@tge#yL|wEtazZSGK7F)N9keS z3#v7C2RJ^S-rjcz@_2`TUk-bn^o&Rh`heZC?I@}tCLH;9Rm(-%(G;2#(9}Ajy6U5_ z+#RWa;jX{8T-laO8^_nD&tyth!Gzbr#V{Tl-N71)^?bHRE_hB)0nCuLIrI^A^@^1v z+v^AaJp10)TkQk@un$hwJ)SprxPR@R{>b`K(nz&l{KN$vs**-h-|J;C*Z5#j+F?!-A!Q`YIUXb zHt1>>e2Bz4F)<7e3`&-_*h{B%a*o&1_f=b#;%J?X?=mp2A>kn*r^d(aSKl79HZS%9JTpLE z$t=aNHDG%n0>2tFU%pWr*xtGY8?F2vclqW#INusZYX8eDX%hOZ(mA+U>iQioqc?t$ zLdU7N7&>?rhpG|=)+FHAs1E|*2qtS7f!!TPV$5w1^W<|Rg>V&z9N2;$_zcVe&KG25arr#Y9-MW|qPEiXMST`*Z z1t-N?yO2CtXbG;Qt>9K+WXAY&>2ca8da`7>^oPtjL&Ten(#jZJ7I(d^CL34f41Xvg z`k9NmkLD8F`s$-LFX38WJpuF8gwBssHB&(X?sSH!7tr-kQNOCB3n|AP~E zkNrtxtY3*B_$mQtKLe6LkLgb((2PsVW-vWS=A#uM{>EtT^Zq)DZ9^%PP;2HuQ&;KO{PkMlbvI$MpKC$8Ohi%dryu?}SGS)&>YP zvJ!SlUv)0T<-@`I#>!)&|83VnO~XfT*Uj;;dyKpb@Og|hQ}K9=XyJ7k&e39b8FpDe z{`N|r@ zgQcr6V(2dB7f%Wj%qcj>L{jS+kOh=>xom$l#gta`UBhAx{f>dN`Bvnj1!Dx3Op%4e zKDSXlpQMLTT*NVi=Yb!ywv+Pb;O#rM5fe^)-I7;_id`Iogkak`uB~mO0!OMCe>^C{ zwC*Y&q7mG6iGCNjr(IOgJ+bDu5ONTAnOQ)W*KM35%_j;jn@O>Hdmw|h_qWKYv`C@; z<(8z$N0@qXWfg$a?K)9jS@}hO?5g`@W0h0bhz{h%`8vMemlhOZc2r8N*Uaui)O#SA z>U{wy8j_E;BgnvMJG0KAnpj7WPgV*peTWhMwi_Zu#Vs^h2VQ0jXsykg=Gi7_?wOzc4}$%%{ue?$vL8cl1Bk2iBS?x7+ilu zCmWM~F}AXy-nD2Rivlu9%dQ+)WsP8=6I&}=Q+*ReR%RA8xj}m)GB*m%ae2_HXSm}K zHO}*R&Yv~$1h`%{OXlgF;@%vB72Re8yE@j1M&~CSsA}(!bWY`Q7E`&^-Yi9O$m7R* zITQ4p0BJev)+&xsNhThb#=RgV*d_D^W9FNtXxDW$d*m7t3zugf)_)!XG!NyG@5O0l zf7Da)k0!qo^B81CI@ZE>a;zN7wH!B~ky95$>|fH4-M*Lav~U=e%(QrUiFj8FHL0!^ zzb((V-sy5rlp0^|BC|YPI-B@Feo(A;q}vc5d-6xyz7g!h0kOFYC$mQ7iJw%DV}5&v zd>>b&R2R%xu7F5JCCVBw*Bd@%L|sB*8LM#M;V3F}PMswl}4Edt@{;8ksB z*tBDL0Tb&oaI@p=n}dM*tk`XgV)-A^ECK53scmM6jmJfHj%S4Z>Lz@u25cEh&e!cX z3dW_e+DO?8om zY*etS&@`fq+`VH<{um89p1BP9zHv;oIvTWfijZ-)Zm`SZL|*%4a~%|RvNCjTbN4WN zFY!*c?$_t{VNJ=+#>*!x~tpA-6O1EX)seb2uK*{P9O^W@>()< zb{<+YyD!^KWuT;y5)Ihr`gp{&^y~f+{Pzpk9vg;6>SJr75nAlhdIyTo87~}r01Cuy z;xeQ6ZrK*jZkpcyA1Q4b|Nl9AfuX6DQ29M#nP^8c-d}hlQ`djh)pG!L^bh0|s;nrw z!UV}0h|%x6zK>X=^L`ui>z3HnOj`0ty^RP{<-W1wi5)qganybF2cmhkUk|9V`@M|4>|z!5sx zaVVRAm57mb_E$h`MflaHIhQ@r`LnTdb3x$Js6A4zj{>5; zISvANP7EkvMJDnPnP-)y9bci)o{UmEs7kgwd!&_Mx?qfhTq+8pKWKE3%gauP@X`+6 zjiBl*X^p8VTrx$$cTvG|v0KGvqV#aQc(G-q^kDl<=kQd#(}5;&##p_h)ww$GJh(Vh zWuNB)JaNRBk51cO+zB}K?~uHP4N!b17xaP3cpyU*dOZ0|E)~) z4tk4W`awC~Srkd_u_c@I4SM5fK{*!a|K!4*8=T@67uo)z3pe*joH&8N=ke^52bb(A&o| z?d(4fu0-Pa??|oND9Vf9b~8ovuasA+@}N6U``$J1b!ZxIEU6}88qY^3lSb{}8~zJX z7{Occ(=Sop@P<^MG#D?80XFp8^e9+Bkq_cvXze$v#aqHd%S7}e@FVh*d?*ts;p ziVugYdXAK0!xlaI5lmn+dsGiB>U*pLIAKozC{?;sZ6~7`js61efGAXE%K8S^>g3m$R~t_tw_jf)I3lU!0JF0Y zW~bCJ@^pl9i_$XGmS=|vXGwSmKb870T7meYadQ}Tn)zwlk$c%KFBz+oiQaA|0H|um zrs>(n_so!wvPWrSG-L+}sVi7$qUQxi zq%dC4zE&;D^$MVD?_;xUVD3u30Y5@p6gek#;rUc*%kG@10V{5WzaNrw5R9|;TUD

1MV(kdEFdqqvyoX|AV0-gqCrfgm#J6XsM`=>T< zZFK}9jzul){Dg2rms&VW$kEi1E$JL(DiU5Ia)2z2n*yDpFBnd-Hbn3%j)h!KxO<2A1|=`pJ6(4<1U+UYilMy!3@!Bt$xc!7C6>~1VmCRTk~=WM-Cz)&QU z?udA?N~czJx#IZ_Avs1X+hc!`W4twf?IL(0#o(-i%%o%M0Ku2-nm{ztcBa1&x_?bX zA!ui0oF@j?9S%2Z?~V9ZMpIE55Q3lZgyhmr$=c~JA2RHIgfvuY6~XYYPgY3AqdW@r zj`{tYy=ikja&Za3(mR4_#AZwPlG?eC@=dkK&X!yl7G=FH*}y+)6D}%bs?r=eo+fTh zGXVHDy+|r-w>{`|*{vi}QyDd#EzPoYF@!9wjQ3?Th!603 zN$AqNw}=6}s|ZIuUNLPKo;=uFeN{%|J(LWMHzB>jd`|qq;D8E}bskj_vdzRJ>QA$w zdFGr(1a0l?$U)|G$oCZ2j8&m!bMq z*j`T2F=hIs>h|aDnd=38&A;7!!*H*kts;gVZm_?1$A|kT6+OCNUiZ%fYwj&Q@wEHn zlOFcucPhqRwA(n$2M&b6m}%lR=^y)Cg3G*& zFDNsZOLiP%{1W1oHL46$tdOf#&{Q^0s1EjRm(v9RCoF)MWE(r6;Mz*7+Pf6b@yg0F zw=s3$S$nOZ2o8A`urb?y(az*C?Ac<>HzowRs<9X7$*<0;>zh8@;VJYKfw|P+II>q- zZGR_{d4v?(G1{d;m!BX0xtwQi*98kG-cY5Zygr*>gmDo&J%jwyVjP52h{PA&4Grx* zv#Vo4U=x8|(;QU`fdzw2l+89S==v-ds2czW-<0VIJrg?tnMM`h127a{I8eInpB04~ zjdWC50E;;UN;bv2LW+4|HAnPU9vxu+VSVnzR_?be`&s?>+b|xM}Z^Y+^Yjv%;1CW2u{A zW~Z{Mu-ZKUEJpv78XuZ>jQB-mGIBBF_U4utI3{z*5s;GgX{I<=;mYg~k2rYBKJ z;=ApFc=H=>=7oKd%u8NA%sGgE_fLP?q0WS%G^B`W*P?H>W79f|rpwx`^TxF!>L9<8 zn8J8-yNVy|ya%&5j3oa~^crI)kHFFBg-w?G)AF^TTS9NYUX?R8zVEG_wU1OTy4Q!frsO&F){?c2RGJpceK$>s)yyT|$y3+|eZM4?9R3*7SXa$KPM4 z`!Dff-tj{>FH)F7XR^8v@04jD>1e@1O`04T&e3XmhAsXA>0!tq{W#Z_qkW4w4@6 z-nN^qFYkCBnAe(^;+WbKZ9lTTV1C$E_YgdP6X8}~EP5V4sZSkmHz4zrvEecQ{G2=9Z~x^&?}7>f<*e+r z_yY(qqHNshp1>=JBAXH}tJbe8Aa#KQ&5F<&<+--=pj z>zAo|Bd}Zi%}#Y*)nx;7`+`gQUF1|AKfA{v*1XM(D9Sob@>Arrj3%O?BEN$Wdfnf| z`(f|AkRR;U1W!zS_XfIzD+x79)u?;_v$$nrV4B85Q9R{-Rp+Bj=MW{I?x9s}RwgZe znFXy0I`LJx*L_t(4zsMbOG@!eJm+-VZT#8%$uwb=PGFXI=Bn|j<0>3Ql@GDra;BssX=tBh2CK zz5?e$YvV|$95LkN97C;$y=p#P$%_Z3e$A5^BvUoVPa732Di~RJ40Gn`7Cetbn0Ar5 zhS~+uqFstEApnlecB@ZtFgORbMHnc#~=cQ$!2A!0dBK z_WXCVY*Y6OnZDRq-~o64EqePEZpZGK%3{O>&t_JCu?|C=eZ@DpxxY1_auL#1eINkF zt1M=fD`LQ|nz~!qh4#cq9HE$*c!nRgl1~HKh_zf!CWY3K5hcXSchnu3^`BxX8oYkE+F^Rf zt`Wqw*rc2WY^i%8w9JZPS?&Bj0}3B^jpntQuT7=E%EIuwqv2PZss9oF#SDZe^K#Bb z>i&PK(BXH-DeFoBj^&&SEdSwmM|$f@0W8+Ix^JzWk^la+eL!?E)6`h^t!0UKG4rQU z7Z4m=_pLSi-?r|1bGhWFz__UHTT2n|VkY|OWcb~&I{N9v>c3xJ9~fq$D5K?x&ByA| ze0|AL#F&RT;0@CgLjlml5lpw3TKy+vj)k?_Xf4yU`INCvgZ-b#-Zl{nwHsMRS{j|Sm%#e|D3lUqc?ilNE)w{=0PGT- z3qNe}@njr;8g*u+i3jB5l|2M^Q%{@(a%M&sCi%!ByuTf3YvON4Rv_GtdWVMuNIe4r z;rtaruQ3hyMPYidHl%3=PBoRBFbU)0`oYyYupa_LA&D9+<+egN4sT*7ANG7i3ob4~oAyYe%6MQvjLzE?&!Nb=#0gr6?mnzndFTY&>fT zR$#eLJC80s1E%TU{$T?TAc@qjKh_w|%P|Ac|T~JccR(`K-{(g+KSqSG`(R&Z6uoLkgUE!r z&FK)woD=QvY?AVmTR(=wsI< z+wa{B7wWFc#x+V9)b4lj7;OHtDbeoYjTN}Q#ntbVT#nR)Pz>V$c(0~Da+waQGGCvS z=}h6m&m`ulHmEWd9NUS2~tz+g639Iif(e zK2Pq?ZZ3MH`G|N(UBwK1>M?Sp?zEZugxFjf>oAPm^V@zySt69i|5i*f!tg0d@3>4%2=_>;c4(5p(PHg*iybqFG9xX($Y^h>3UhMA=Z2el^0A} zIqyAmWT|LkuwsHX7i6ZBbf>saY)QI9G;GtyP5n?h9T~ZUleQ;F8|RY^Dm(bhOD%%2Ar({8G#g zqVTBVSxQy3Ho=9vxX9os##1?Nb!#1KwCr>`s*yAAf`_0&hM&lG5GH9Pu??43TfP!)dwLU^|i z!OixCpDK6q#pKcZC3ZCp!Jd@PR}q=GxFnXWtk9qP$22I zDchx6S%Ei@d{hjWZwjIi&SymQDBHO+Jf*A_$GofxP@J(+tUp-UcS@;aAEO;fkFYGHVTT^zFrINuR(t7&28v%3cd`` z-2gSfxI#vdc9mP%<5e+?IVRt<(veH))s64Z&*{-+#gFZBZdZ#&KX;Aph;o~%FjJ;C zBkq0d%T9u&AALH*>`TgxeuJfDo)ir{?|2Dc*AQFa1;kw}D^!~0X*P=YPc4Yg>7Zhw zT^>v%q3pXYR8CQ3G_@b`ij!N+aTJEMdA*L6zjzVQ5(ZD)&aXy=9VA`*I|r2q$TYL@if;+3~9a^!K?8Jp%z z5_LV3r`dP%O+C!QhpZF9YsPwaCFuwl8C;qSR~&d1t@4B@gEh%j1~(d{{uxj9*~?aI zZw+SNTTKpljk`(G;;bA?nMDE;_$+OZwmDW7`#wRL&4%1dUxfFSa}24oib)L;QOKDJ z9@gRKVxdb`sgE+@x*yxGL2Yux><3M-3kk(Jf9dPG9Z@kC?|XFXssd%rzkifZN(14? zb_+rYe9^D3TiWhw8pYW+;vSIDXz@n=rH}v|)`JJL01U#oRg*Z8{SM9kQTWHm z#P$z=Q>aP{UI(2x!i~hTIzePrXKg!51~mfDidb43{K{VcDoDp%{=LmC#h|8mhc4S` z737H85hqh?&3w(V#T)rEL7YxkQ`W*n(ia{t78{VB`7VN120Zn6_YFIFG$3ra%x z16y)B6d9tM4U*ilhjen-+-EHS{13K}VqVQcyon$WV0+X1ki7|7)7s#CAdV6I3XLM4 zZCkJI*%Jq8}%Vh%NZ<6i}J9li4Vq zbA8FaEVKG;AB`?GiZi9}8ev#(iB%B7?*zb%yPPynuj^Ep-P*K3&2$e}jhQFIz@~Zw z!q(mD1>zUuubJ%izQkPgSG2R~Dk!>a+Ar@s1HEj5g8zJ`gJaBs54o6`+HtraEFfA$G=KN)$~%EHv+Co<&URLG z!cIHTgOdxN(#BTJW$jk}nsfiUyAmx$vf=A4B9CNY%^)^~#(IPrz@d}_GYF7K_t{$C z=*32v!m*`p<7rk&F4MVR++=r$U%7bYuT4kEMkLmEOV9;p=4ax#OcvB6{Gl-~3Zg%Z^>AZz_-(pGzl3B)hvA^hpTrCkXu zC)^G2@Jf55Lz)S!tTM6&#bK!U|CWI?fNvgI_yR;%&|J8J`Y&Uco47#55=gW|4BOo)NwGI%ks4 zh2o^scKhj>HzC?gV0U^%oJ34chp_r`bb2c_MCrg+6zlyRYa{-1-$5s+YjU=X5bsv- z8$V#R!rd7bjef~6&ZLkm3{uAvL-3R!?l4;df*C5RLcMc>T;)|>ZQD$tW^J8q2v#<* z#luP=QpQ^0lDgS8cOQ17r*3(M`Z*_Gqsi}aK4;3PNwK>%o9(nNXupxh0~9SY&#}6E zA1pihiG0_FlAz>dCB?Vr)y>6+Pa!rgU~ydxP%iW%g&tJ95vmaLvJAEqB(WwHg9n=9 zo56!nC;zbXw9o9ZGZET z4)65p3x8?+rIvg@1I@GQagL%RvD!J8XOtYg%~eMJ-F=?=Zt|}12X-uJmUy%M7@N`c zU7qb$#1F>_gb6(#ArDeaT+>oxGd>Qz43GMlcxW#%mnVMF{^JwZv0?tnF*)0q8;8;s zQCC4LM2<;yO$tS9zceeRZP#KZwG0*lWHG>%2HQ5y>-abiJ+ z*M3;>Uzm7`$7l$X&%Q$M^;bv|`vQvqCEHt%5Z0XE7Hamth336{mL8iLawT;+(9fl0 z&jkW_+ynUEbWPl3Sl$l$bh~-JK75HjAE`g_m~0NBq4|-SrNyX6-6W>WI*W*C=#WP9#v%@}zu#nXBupCxyX4i2 z6bxdiMH<39rZTZwYCMM27a$c9f-^1=H9`DwT)vKJJmL!BLR+_Jlvc)u(uBnM{$JR1 z4mLM=bcYYd-oC(^;%O zwIW@#4`%0W1M=|aC*jS&8g^rc{TSo|b81=n@~|5wa;H2McByy?Ev2W--BR}>60SW} zx&%ciP@b3(J1KL^VoUn@h>^pZm|pyhKB<$cak{Cr@VFR3(pu+7=J5~@NTC`sMLEL) zU9(#LUL5xUHc$0_QV&?w9ioFzk#{HVv6X^al_IP~S+lyf=@%=uV81S9kCFzRHC90g?;S0Rs>@a)!& z3sElV!&5j{Y(HZzxM*;GQn=%z)&$XPNCcFWJiQN5pEs#Xv$?+gHXXd8LI4mMYxsqi z;WyO|d!UCv=Rh+UIgb78%10Sb)0}uR$vAw;GC||NU9aMRRWBa)^LrK9wG`j7A`z%l z7Guebfo*8%WfZYAS(zQ~LRIYY_1|g|*@8)Y1F>G11|3qVMg1J)VbK@;QpOZzN$bP# zS@oJ&J{if7_XdRp`rZPVB0@XD7pd^MxWvuwju6+>BvV*PYaL&M2jsFh)|W}eEyB~| zk2+X?UJjw66W~L|f(I!2wTt>n62WuiU^lhFyy(tu93^~abk6aJ?}j8jfAKoFqsvS< z;Zki4NxIA_tX89{{SQ0U7o(MiQFH>p7-7?cQ$w>d%W50zNG)_uN?>Bt52)UHfiGs% zfxw?2sY<(q#c@SV>RSql9N*^j+$9VX3#J=7GbMS6?9XgHsdGHsYZkD1AMAlXJUtgYkJqX*F{Cxm#Y}(6y;6*_hi&%Q1T9KH@zZ(o!+CyUkb|#6NI+j0TM?+XgJTnA7+v zn+6H(jqRnN!Wd>R;Au%Q$V>BBq>M&^RR=WIH-w3op*zDnz0mv@A{$qie3H63_kDR4 z;V^M3%A9pQ`^Eo)Br9Oq;l^W$>vyH%Jy?7MNDL>=1aW{jE>YMjc4gJ**VW&A!o}@& z1b=Y^GL6$AAStY+nMJfX}bX`&i#bdHac8R97RROKD}2iY_(c*F`Q z)j!7f1sQf}Ptsi;`VS-xv`ERMb#k-wvTHQXtWK$|i>vn8k3WVmVxS!JmxL$$(_YM( zZ)D@l-tQ7XE-JB`sBw(c6TF$0-xzdvyPFaSLrTN-*Hue$$OF&h6!)GJ7<~n$`7l4a zSi1!jK`XESBZo66sYs%nu$O(hQ=7G|cy#`F2jS5c_nWVv<PzhtP3xFLw7+mnmiOc1m@l}B3WAU)Vkk|hOUU1Z+b-TfL;ir|pv>7d8WRF7P@ zn_s}0U@mkz95Jz-W;4YeZ{}6uY(DC2(8@;^t|LO=eWY-fsXaNn;|_z-bEj3Y&F(7N zLX>C0_-Mz-COy0HOy5UME~>A9f)C1w@Ae-|x|5+mlUXx;Ca_@dKM&tZ zw@`#?9Pi2eUuaTRBpJxNqIZVh)Bc4YTB0Th#SdgP0jAJYX?*8gaZEW zcaTpp!!usUr~BgSM0&af{b#28C>I})n$4tZC8k_))MHpH5u+FHSLfEE z>txw+?asFtxqoJ;Lb6}PGiEz8M zeTtTfD!cxFG-)m*{&K*rr|G}u~4Y*iro{ZGs z#bXve5BDcC#@(hg^>6#?rBtTN$HEdvc7J}u8Crw%tHRDpGr|JI5G}Kn?TeMw#7Y&+V!h0Qn#JVVLB@*fX7;^)nHVCBOQoyLdF@)u++kj(A<2Tsp z2^7ZjS2jDf*I~W3LPaw)TtX>LjgsuBtb0oh&5~L0YPhwyoS2GIvqUaxZFR;)TTn(W zd$mSPdx=JF)^ba52*2J9XqA~&n21Q%8Xu9aP1++BuU zMD~cqa8)G&Z2jk8nGkGsIyBv-Ku$hyPS1pFH5eEU@@C%apb}m1X;SDX9 z<`&^O`-_Pl9T-wo-U}~7q3so@aWkFccN)pxxn39czvd&0LvnFDs=kS(VT_`O*%nHm zP;zO#;EbIo$Vmzt3Q5;zy?i+y8g7?7b)3EVW(|B^ly8Qg#U&)=2NULO85p<0jnG*| zfz%yGIF}&oMDy@oL~MUV0xCCl8)ua{(CGD-i^cjUVU@aK4*kJYFG=zM7RB0w-CUDN z^jXK~cY-0#%1h{?z>tb@)7p|9Fm;ax(y^(cyZuK*=ie~q62$p6d2chk2M-7MlE9#3 zIy&V{N)Pn$#a3!XhO8eFo6=kX4C zHkN*b4^D>F)fW7Md&I7G*%Jrv*p5HxM~HR}h^}@0aW9dA9c`*)!7(Ft71!4a1jjN2 z@fcoM9Or>rr8C3c+_T#(z1HoFDQ5*q6k!et!|h^Ft)e-CQZbI<*DD0L-ZIPz28q%3 zsowUn*7m=}-N;i;x-Y|#>M8g~|6!l(Fhp#xk6Yw zd!Q%EkbaDEy{-+m}>?lDk>Qe2ClI)`on6@+}GMtER%j%(QyzC8O z_TCf6`BotUL#SxnLXW(p10QXqba@( z=Ri5o6oR#sJngUD1d{MrYO%U92}i6Jx&9UnK{YTDmL^xinrZ7HATnsW z;g81NPm|LX@5grU^s(uPS%yO|AA& zyJYp5B(FU>%eWh!EaB4m zR4C*n?$i}@OIlyHsU%~9pMkK9Y*d{}vW1_URc7_DLn7&D;cjaMmdoIOH;&}eZdCIKo*YUU zI;<~2LyR%NN~>z>5ST_=PE-YY1l}&rKA!mmhQ?^3B2A%xb%Ao%_qTxSE3v%K%)jp; zdgl<-aN|xb&;|msKD?L6S%D7-egTqTN$l)KOitHXj+SJbfs4(g1VaSdKD=FE$QC40 zq#z&$y)Sy;fzHho0j8%xRZ~P$JQusDI7ty8Jr{ zP0u!ia@;ck(R#w5yhTrvL~a(YkZzQzZU~Ag5_-U7HUJljWB_bsJjVDPE`?TDJ=0l- zVx^tu%}uLd8yUzuz{32Xd`(DGKKM4Lxpkl-#rORji6H*AV3Gbojq~;m3dG~`c`38Q z$B=+rM4@>y|1scn-FT@H620VX$&u!Jj}7mm+D(oUJ4?G&a=k>|4__|nmi6d80#<+- zz&NKn%l8g<%EdRtvk!_?r%|xnJ;MNLwY$THGrKeInai23NXwMq2@JwazwEv$`i><$ zEtBOO2Jvn#4J3GW8=ie@(JFWR=sYjNEU$CO1vb5`tvo=L&L1dh=O+HL84$cxX*^-M z`FVc&@CYTaUa7NE0~f?>VG)ewE80DS9}LZodz@T|xOW!EVkKL4tDw;0{)#}Ie3vW) zI8*CSa8?m0m7Wxm>Z+2o?!(+rUmwt#KdFOP@Ze{DpD6C%JyYIU8x zTn8z?ZQV#P0MRe})Ype{uCt zl*OTH%LWxelxVc$=zr_u>3{#%k2awk-)ulX5b1wU{g;>kc}z1-G}Dicy7XJ|4>%yb?tK7p%uCeQt5^-#>r3ie&b?IW9W0) zWitfoa|Ax{84?Yw!t_WCeMRPKM!HH40^{cR#Yq0D^5gyKgnGlJ0CEAi#@bH(6@@d5 z1-H(-wK^`Riq2N))h6nIaa9NhHT{yG-*T?%o>}syk9g3n=_E!hj>fJDk>$zZJTwv^?Wfo^=StFo5oN4LDE7RY z%cayB4`P=m7^S-KpoeyraLnR?-zyPA`5C;q%kqNxr9-hh`m*R=Pv;`)G12=6j?UXj zZN)+`CNKvz7%b5zJpo>}t8u2y_p4{blkW!2P@U804J1r(J#5@{Xn1Ev;j?hCIZfr} zFz~0Ab!=j~BF%(b!|~LaGyMU{XvI0;rr;>WgBnCD5VAFo;Y<8m0|fitJ@!63d;B}& z40`*de{ZF&-63R65jarqfP=rAd&^Dmy1>;u5MKF%DW5*FmX8}KG3Z#{m|Sv%j6W=3Wybz& z4=99JDSmGoc~*s$KbhTJ!sV)Wm*pewtnRH9s*m<)?8zs4j#vO?%&l0!Du^|j%|2z;v5sC^S<=m zEkM$ZHen+&;S#aNSX6k2hoRS?bB=kxALm(!x~jUW<)TYELjj6ib?6zm)wzqQAj1jyj#j}AT(F;YXoBD11b#%6E5C~lL{fXBaiXGST!+;b7C2A z0Cqr$zl1lf!9(_=Re?{Nn?s8ESX&Eb^1qHmJe~Sec`D zM>@G;A76&Sm{M_`_}&5~4#9xXWp|coga<|U2hXURg z6JM@weD!Q~#KjhmSCDJ+8Av02sj~f(+^)OSQ8DMrfxaIcez`GPYZ#$E4KoOHPPHgg zHI12`n9%@M4x5}a)t6;C>&E?~KChXHQXF8b<>yHAIWJ^w0XvZq;hbRJw-BL7)Ism3 z8|V3i?1^hv1_!V;7fC);qU8J~uCI&PoRIvzHwr#ecxNENRH|b)06!J4>&ss55UV0w-ktNsH^qc@jgVXX&goqm$P>jpNjH zy(kuZH7#okr+i4_5b%26@|dU~YkgHC-SKrTo2dEcC)df<_WE{yqbiWo`DN#_eN}T9 z(aO@GgD>8xtbP;WDl*xc>3;?rfUGut!|ggei{q^V-qK*Eg-%6!vj6uMVv1YZPRQn5 zqy=%o-oRIT4lfX5yZH$6F$j!Q*>$0cbx{rq@}!nI`a1G4%oN2EYRWBuRm^6=7vBuZ z^VH-AR0FI-S_Ef=der%Otavb3pT`D-a%#J;I=Y46-3j^MUu4-Zj*#6_9=sGwqVr)b zo(Bwx(!=W{?h7(34s90l|htsiIkCv-lQwQ+C0G=uZgySr6+0$KkoUvug8?oEV)< zk8_~+`Zl!6Rq2hZIz0KEL+}AH6(Nu;)0cKS0m| z8TfUwzII5+$UNy^P7*!xOL@GVp$Nu{q*2)jCrqc+_mz{ohO)JiE3-+Oalwwdyw3Z7 z1z+sIPXPjmzfg*QW6y(8sM#lg{^s7(6XU|-Ovs7(9HPe4$oc4%#3ZVnchu+>N<w&`taTp{bQg)Jyv`)SYI&CT)(?z!w9g0tvB^w z8Xv?yQrnJQ{3HdlzGSA;^K((`T=v`}rp~p`2%`pRe-!tL@zM9tx90pBt$(68InSE$ zX{woc;IO1Ev&_!zHN^ISg#wdh8STP&Abq0z^;^RnSX;TH8d7PL3xZbZwN{!rpx}&#c;YMMZ_j?N{8~-~8{(u$# z&G>Ai9XyWfuJ4fSU>=!MFk?3v7PtkDKA|s>P=Kl}bZ#1wQ;wFyK}o-le0!v6Q*qzF zK+B7v(q-`WJ4X%0wE^Rn<-#=wZ|UC8?3wg@I@Ne`n3+hv4(%F*VxbrO3xg_O>=y%Y zD{Kyxkm`1|>mRgQ5Msy-ufBiCNBvJ->x_K9uXkD$Uxv!&wFNPiyR{G22>ve zk+xPtI+i#M=Bnl9e)#nN04zY$zhVb)XWgLg!dA{|6LvM&+1upcHJ>^EcpU;$F<`TaVda8=z z?5gOl2ZDB>S>JrPI;oEUyx9g)68Jm9X-=gC;}w~&IKS!b_OP>x-c-`1Jhs<uK57L-LQY%kMPf6^zqH=eS6TK(B1Hp>q*&vQF%{cd|= zgwg$_@-`w-LgcVi-jP|)ZPzFzCPaZiSQBnVNQ3G50MgRguO18u);-BGy}8(ij|5KR zs)U$N7i|JLw73+`G%oG&Ary!KbF>SrQQVa%%jvzIMc8XSngNXK&42F6iiE(3?_&%n zdN;$49AZ~#nikf&gY}12A1xw^GV&tzS$k1v@g4kPBJL7ce{J8IDLAgUPiru0KvM11 zDvaVS!wWu&I|zWZbGhL0Vv2 z5G%F8UDmK*0$=JQc8=n(-L96}q)MdV%dz>`oc{$H;7hQ%bgUQt>s?C%goG)qXtIYW z?ra%0h|7jKXe}2DoNJ0Nk=@EUSyF>(49K&O0&9;J1lf!#l+)$P+)IQZ-yd=w#Ejz< z>I6erA{M#kE;w0%I}$ULmI7)XDT34WR4S~rxJyKj&g+&t)}I^e&-Jvi{@hr9?pa_t z$NF=@UX1nUI;J?*pBwAXp|Sp4a~tc=jrHgHMWC_%TuU44&&5n+tUuTHv9bQ#Sbwhf z_{aKlEp4nn*V@MVa}DiRP+?)LKNqyI{@hr9Zmd7I_38g;S!t|4*VM-Pb7TFvvHqNE zWBobX#`<$({W;%$1=>*rvMQymc~)ajuq7ji1$hR+tn+fQAOeUY47>%G@cGCbDQwro zt@Dbg2_zk{F!q*iZB#8JG!;ZbA`FI3KfY_f0kJ7b`mvHQr;n`NIXQfvo8mO=b|)E^ z7Vipsk*qLd#A#JjfeB$w2t&Le#;MmjAV!k1WkwKI3MZ?Xo&j@vT!BvQZ=@0g=Vq@3 z$uTuUAe;y;CCbZ8(ZYAuuAUNrtV@H3bbAMInF_%=sb*o9!P#bp>j@N5$P2ahc)NCy za6yGsD9I>c%FfUv>4R~6T770$ChgK#3Nmz|q=BRPfm~N9l~%>+sM3X9N+ffTZe?(k zj{lfZ?XQAi!i3x&_SgS@Ag7VL+mjkNyAqph(WQ;o1qaN?t-gCCA)2e>fsbkDw-m6& z4@645Bo`gF_t=M3>84-(HJsoyQ5eayevbtqA? z1P5s)AqtHy&=SyFRyO|q%Rm1S&DWY`g3PD5!kHTcWrQvDjsuBke;>6-C%=(UTgoi- z)mJz3-s=bN`8XxF$HB2pGrny9W^|>(UKPIK=I*5miOcSTR{YNBK-*U`h_@(sQD&xQ z#5*6(SW2=PzZ0X+1}_jSqqghKZ1@J+=cd9fy4@Q5d=j5dh4UTGvN=v}uK9bulpoot z5M0#fo?%VDS0%eK#d~~IHD6i>uY*7bD`*|i&DLe%Hf&_O6bZSfl`hE;(evuU7n9n2 zWO#+Ydinc5nTLY7K!17(FjhG+ga@dtW`Q}*vMvKnc-7x^Ch^_l)m9I_#nB?<_4VIP z-5VuIwwRc(3a_0V0EarTq^~>FWq$(REzioFyd}3Zw!&)d8_sf)Q3+8j^Nc4qD7Alt zrL-=hD(SiB($C=xYY3Da7lnuP3w&p$+o=aaANq+v-16Q^Kbwxb_ zOuZRH)Mr_CK_h;RhB4_qf=Au0U`CpK`>{2(6|#=fofs8FNGdgz{{c=I%60ZEvog^= zd^s^Cm#Z`C_)sUHS1%9bk2}0Z zSn!hd;#R>F;!RK=HYR<}8Szz&dwQx@vj)R6W$4xH)yt?kT#}>|lHQU_oAx&_m7Fau zxRi9B0bWXRbZYV!Z%^1;>tW_xgaNKSvp6(2wO*#5ksed~<1jzi%MIpt?wOyV8+)0T z%dx!AoBzRD1F9s-)FoMkUuh+u1Vug%L2lhRtln7if!WVvW5LDdHQeXUY~S$;2;pjX zRl8;>L`5%Gn59|oJ@zZe*8O1=Q029HZ`OLE&S;dK=) zLA_Q^(>AB=iQv|pn`h-G)iX~2lqX?{Uh7UZ=2V-R#zOo0L;anx!Rs7L!id1umf*ap zHpT%<&Z2Hw&natC)6YT7*Lns6Hgm|u#(S{ljwWU@c={W`P3Bq!k)?VIt11$F;UCXl z7=$e6k1Z}*xe!F3X7hBbYtnPfOJyn=llvqqHO1V+t%xxWctz~r&Nk3D^P#2z zo4nvHZGAwoPv=c=yS7pybfzL{2q5jMEEhtQEU9ld!9(^BgA#UWEVuMzNX+y2Htw#A z*?QeNFIxK*%fLJb2_UzJ>p#)=)z?uDDSh;gon%BPQ+Qw45F-oW>KY2h|7&)Oz=gu1(X&2^j!D;p~7Owo(&`shJ1n*rJvCzOkj=;`(;8BHKu-UoanLZf zg+tRDXT$}J7`e;VD6tYGKq0F9!qF#x>as^Qi;L+NSiomy+ zW?EcmqFW(ze~Au@ApcWS7h2?%oI|=jBMZZvj~aURFi3`QR;(Nl5@e@}N1<1Oo6(!{NF>aEzzqie$jWwl9UtriZhOP6Kc! zXQXS` zUsPYO=$kZ&?hV`kgaT}~!M^RfG1+=VY6P?3sccq4uU7;uc$V=yz_Tloo4peR7_9+t zRawO>%sM5FR`cJY7dRO9g_-xnALB+6lG+qCJH{=U`*goWF{=Yu-4`Br6nn3x^Y!L9 zFABkPqE@7oQp2>^t>j z|3z>CqbJ_&r3!9zvn~)z?NE%W7N=I61h35SlF}w419{_ytSu5d#|4s#iz+5V5XnoC z*c{F|UbjOa^!ClQ7OIF~I75o_%;Gk=AQn5R1SPiwVKJvlV6pa@JOPrn?x1wK{j(%u zZ8#_kS$^RkI7FP}Jf#aDq?UlT=^7ij@@80hDT$aaOPZ47;BXsz>``b9;d?gM;sY)~ zzJh-AEig10u7X9JZX}%KVu>037qN6yktG-OUu_rSP$s;vM+MCP$e#z~VKsgsY2^Y) zIVVVwSv>=UGr_Zrh&F4l=^1iA?d^!o8SQeq?f$X2(PBa-I4jHHW9ON;f0_aQXgkn%C<#@>00dPQKG+qu#|MkDmhV0_(O)b z^kyeb^-2>$i(K;|hf@F2%4(q;fs(A0%0=5%qz&}MhR(??Vb-jh;~N{>5%le1Hf^_v z{Idj>TboTozq*foYIdH6^z7jOWAELc+qkhk@!#)XfzsS6Ng0x|leu$eyt9*1Y&oM% z>}V}Hsr|;YYp@$6G3o|7jc&=zX8-qZ;T!;sCi{WrOLDGyswNiM#Pi_bJU_>1WEIOX z%moPzVsBByH)$?cAmgt#*)4iS3FU3!O<^n;Lhuxr<0_Ovb1^yA>Xm+h{6PB^{W|PIENlj)3N|-2)RTl z0Zfcu(V@O_D4n5b^|E&U`uffqDGchoMRC`=QG2^pDhP=*>x*|8lvUXg30j**CgVv1 zRduBS5R-TdsN<&Uh`(2ugnU^z8#H7R8L{5kq&k~-0LMYN1OX1FKKD|mN4sW%`i_9^11C5fKKR(FvyD_sEdpiTA6!6{`bj;KmIm8{rcqO=V#-;{_~F~A13gR zFHXNc{muUI$J5i3lb^qP^X<*m#XtG!Z$FDNy~V$NJNcPi{4=nT)6=j2y=6cf;Y@@b z)bH^QszO~MB!zwgYc$2!6i8=Dv$F7SGE;ygP?qe=U9L&G#$YpcomHj_b+(C)VsG6i z5`2s#_|D2iD@15qB3(pi8EP99LhPDpohu?!UKB>;r#=)vFKPil=1@So76ng*2Qy!J zn_D{dBCMpAUwE1&EM)>sC}S%~fx%%?jn5cp*{6uH*bLz_I9(lI=B&^PVpDXsQ{kR} z`D|TV@28s^C=6(kUh2-}1%>=?h&Kl#{7mK(Kile9cY55keJ)6{l2>vzFB6(;3dk`r zoP0E=C{@3y-_qX}ryt2l%9Dg!pPl`0%0*c~)OgugW63+^&wjIq^K?-i)YH`axAH$% zr(d70pVcm%PnHF{%Gr#6+C=Ts`WrMsGH9c|<~f^l1(jC-LKMSYbA>B!Mfp;(S(!8~ zJD-_IqYGn2egE!zbd+e3_0tl4h4(oJdk=I0V^Jmv43ALThMI~ZT2u7f=ZdI;Cy9w@ zotA||RktAumA=-{G&rsZ^~%ur>oXy*r6wAx4?i%RQlVjsd^BcyoZB$Nfnrh4nl7>c z#T|c&=rT6d&wVH=Kw)QevoOOwp%ZxaZQvvxzbbgG%I~RBOtYp-$OVlSW>hfJIIheb zo(tDAdtvo96FY~s`#$0_jsIr;t3UmF!$O-H`(_Ows>~ns$1_$$XTCk-%&h-ey=A+6 z$NsyWCFEN5zyNDmlZ3D8wX`;*k6Jmdg-l{a07V8OIBXY$7DdjdaQx6nnWlB@3sWkz zdA+$>!+*EISq&x0hPt7$>^x6p-dWB$Oe$Gv%9^~+&Bu*px|t6W05rj21>3Iiy;Q*N zSMtNFUWY6DL6xo(N#oT5)L5t|w4(3Mt6RUR-}YvNHK;=DuD6k@M72H@E_MS|!hNGmZKOhV?mU1Qe;YEbK?ps_x92jjbz zsSAkLGco1_*s|}id^{vcE&$IHvLo+x!wa6_v809hdD}68&>;OIA33nS@cDD}P{@05 zFMNCX%8|VO>FaA)&6eMOiN95>nDdxSP4*Bg$cbS0n91y7_UgF+whuQ>(@|4MoPNH- zo-&r(vdU9#1eYJ)9h>Nd{C=F@kH_P2>trttXy2%|-YyLY;|Nw(UGuc6UlhE>(oE!_ z_gg=lV4oGn9FCVjU0onJVF#t5IpZx+RS_dOLu!WbjFwfkH%=#I8_@MB(=D-2nZ(j6 zV5z>6@x*soz?Q;Iq>t*dJt?Yo6pb8@xny>BWJ+~MZs;c~1kswyacL+EsuT-p>K|*i z&awqdSx%FB_XZp4h0%3J#tWX(tScY_@Nr_^YN5w#$pm@m19S9)cqe7?hAX)JkD!&K zbAaeOnlz*q*zfwXs`oqMZBtFLPBtwZ;T4iFp~{?fq?HgqwZshT{pBn2jGRE4nu7?#us)iv&Ii+TbrT)K z2f8ORx2f7fBC9jw*Zkkg11_Zbvp*(X+^jvN-xYP=mYcob88`FeeRfoy{j=G`-z%?E z?0YWCPYAxbRRj%~5ig)M#4!A1vlEQQysjg4bBdmfMvEuxQ0a0iVwtXuL2nUj$GWXH zZYVXj38yVIx&<}1DR?V1`q0!EV0CNGm$OejZw@T3T)7QJ6D&mOuIk=hMxL;m8X=J@ zGT2P`PNKDxy20xoKj7F0n9OdBwXPaF*vTGl>}BgExHfiuU)IMC_tZm;y~jYMSQN=L z+kv4TZ7FfCUp4s;=wQ|UzurRcl0BFcz_;6jacCQgr^CD z1AyPC6b?SFr61eMNgKM{slm*#wz7nlB5DBjz~}HJP+Lx1HRq|JTJ;ukuVWx%HMX6PW2^Ls?$YdgNRH?^w}uLLWIXx_ zTeG)W|Z7ZJsLnGG~zcN|Wo1 zMZ1vJzkPFkUT+mxok?tB@I9fi^%UQ;=|ak`ss|@FFnrF7MLC^BGWBA_$BNI@nW5pa zp3*7d0t$Yt7GmkhuCoItksCRzwF$#C*TnkNfX~5hZKWx#??iJdt$k{{(K0Ho*1$*f zn;i?VI(M`^Y&9{XvTLbq7qs<{syQFaoX=`iw>&5*>zO--O3CJ@o$;*IbDku4bzDKk zP|x-PMga%|aG}Od(5#KIi_z=nggZ=OlZB)g9a^R&<)3u_etX^+G-DD{jGMpi1f;yI zR2M)A_M7l((FyqCEi3M2ev5PV&FTgTsgFFm;jW#3YyiD?QV96UOnaIk+v{jIzB97o zKLbr_=;eB;MSUMh@W;C0pB6JH9kH2Io;EOVJ9d&>tYa+Da3*tiNR~u@xaT zG<@Yim|D^Tx)LH58)0-^|Kl72U29kcD@rqp|1rxFxn%KVjfC&4nEcV8GBsuxTq7ti z$Zw>e^D5IskHO0;!;TbYjF~+15TMZ?v2$Q;H*^l_IG+oEGvYJo1>+H$VJtLZ(t^b= zV4semn`6M(Vl>8hLtKuS&zW-AI2FlLrBx=!74YTv@4g?ljfKqVd^57dpStqjRbq&o zLA?&sc1^CevPn;Bi-j5+49XBAk~ctFBCZ+NRpR-inIm2mLlzH1t znX`mZ#U{rctfc*#fq1%BWZ$rx<-)XqRWGg$qLy6U123(oOWmjesjscOm=@yi zK}A|-41urkIGES0*hiyV)S?1#mr-DdtBlH{BjaC48LixJ)}HUh++(LfULRp<9vYyo z6Vt>gkvVz8(w+fsx0~IYNy4Of#eZdx#@oKBo$2JE3FUz)b|s%Y=2=o_#`anWl(Vhx%~^ECO-!P{r23{`TCEx zW?svjXa(B=kE#&{z-o>RUIT3K`Cs8yZbn}ImI;>gXk~UisLS|lNS$ss#_o2W;~>Us zUu}3@H%Jz{xVwMyWz<5NydgsMs*89`6Db(E`tc={JI>`D2hqPu8tDztkU$2u{)pz( zjYJ0(ho=nulU^oNN9&LoVwHfaA773LpRkGj%Qb?y%zAEfteOQ>)ZH{Vs$lrOqUpE# z@o#!!kc>V4o0kgy_m5~o?P&bt%KN@b>uUOav^ZP1Na|x}Z4Lss^+mp%V8zRo0Pmb> zCSqN1wkWhFVp=c(X=0uALx?qgLBj^jM6brltb!aEkwQw6NHOoJTy!o7`@|KTXk1d* z4{9J1EdYw{Z#A8+qg9yb7eW?aIBNFXj-U+RNp1_w05mVe^mNupGaEHsl9Vg00ZcS2 zL}uhxQ=t(KiUPA3(cyb;ao0C20ANjkl7%)SNJoNZG!ZOfO3{3Yh6--s zj}3eWI8?f(P6gsOLgW|1EyEU*X)bS>aEOqdL|KR3tLmVk`PSrxDzDse3}o@0yBoUZ zEZr8+S;3(0f!>QLv!JqlRq@^#ad(8~UAu5RY3IIyHC|g$=ly43O-JfNOWGjl;fz<6 z8#3;rc3l1VvZseS8ZR$2<(XhO*g_NQVl4H#^U9da4G% zAJyr(BZC(j^cds5H4u}A<6SkCyhhU+69vzmyRE;%pV^&K;1Q_rlL;AOMU`)CZm`~V z3nRmrRA%R&Q&6l!YbTA9=xG)AT}g9O>6^HF>%xu+G#zO+^+v}1uMXYFmFDf~vF1eu znl8RTQdec8Uc7BV`|6S5v!Q@n6;Zch>>qGIV;Io*d2iP}HL5{eXtbPKyADm^}PsOiE798F}wZ>^7dH| z23>KmKf2lV7fiL!7dA!F2j;1J$8yv8e*Lrz3% zLWlnVqr%p}kJqABm{$he+O?|T73;JD1IOgC3J*t>5y0#kwe28JmRx1R18V zz6`_+BngXc?`Fd!$W1X8Bnarq6SzVaVioo$XIVmxRcKa<2(wRkv9yBHnve@-0mR=4 z5zuBBN9C#bjxa`;7}5~%XF!kj)ib5h?LEyKDwnK3kxVmM@F~|57@?HoizMQ>s0U#U z7P$PmoOnyKCWxIn0nDq>rb2j@3B#`!2h!^(g*9rYXn5CW%X$#;b_S`ytZwkJXcpQw zS{`pj%xc#sZ>Q$TCc$I9L60c}I_HcLo@(`hio!wbMu4OR4q4K*H({FCPYcLEtlB}- zVv<0_(sQX*{I$@j{}bMX}f1hqW{~)2Rj2RJLZK^UThZ-d6h1@5dpX+c|c@(nBKDS2;->pQ;f+^0cWD7qJnA{x}~d6F%>etoSD2OwuF zD^eTL%UoiyD)`Ax2lu4$J!yO`&y&XY7-t78V4vsjtuMROdDYsg44to^I0m95ZMAPv zU}yJqIT4uA!NuH~W#8FezsKUsImwt+rcUS;HZzr|#|boF(qen(fo%8X{!55`q|r zRxGH{!si{ArLw`AWq}tY!WT>#S@8M7z8ukvMq2UBStN4@P!OW9se%@zs-z>hdck{p zP~V>0&m9X=U4_|JpCMJU)_&U^=3IY5gPxmz6?}`4#~GYje>MtLU41sYfcI4xZuL2A zDXZ0IkR7?NklgC?bY%W(9hp5AvRi!~q_ExUbAUp3tIub|Ee5=-SOO_m1BFHVGgHzF zfF6G-S-!+gWaX!rGKqo2XYagCz*0B)53J zIEvwK{wG@U3^f_OeHD6ugQKs=#>@x++khmRW{T_PzVU%`Rb|X2w`@5w;i5$EG_@}W z4+0~rkrCc9bZcPyp+|lgjp1|4S}J-pKeZLsqck=LTtEGR&8`~Is7GQI@M`F!$#Am_ zK6aNjp8fy=^ZWBvQJ|Y+Wl%b$fKn%r{|*TFvw;MFkAXF>6}c2d=4Pn|b?9&olQ-5YRWM12h9d&KG1xpov12#8sciT{M$e7A$!RA4}_UcJlK4)6WZO5%^dIm9c zGrvv9ZCHqH^=uVvYoBnt8$M&r2ZG2rOmN5Gl+qMN238(xzE;)F2&KvMP}5PlM|l6+k6Dy9W{%SLj=WrX&l=H z#TirO%Y>p&U>`>A)jk@rhP-zwuZu+|J-nSZ(gO)dz8G*=vFUSHa(Az5-rAX$0~i}9 zP*~$1n||P>JaQGj4Rwqc@Ht&yk~rsgkR@E@0?6DXO|C^i>c73}#Kr&-(0Aok&zZ_3 zMhiAh&d41Lz{jgfe`jT$Nq{_|VvTeI^Bqi*iOuZ75x@W5ZNu`%aK!4E={DELk(aDz~Im-R;X#?3Pm*I25WK`NH^zKmlnA)_BO-d46&G~ zXu(p7qCWRRU8u!&q#__ss{eu z#tbH%L-S!GMj&Inrnna1RXGLMsP*vo*y>-pLu~L{dXS@E(v&4HX%{Z|-f011e8+k- zx1xrAkdhijTo@RsHx}60SxOw#bIuZWhln{Ht3!or>mBP$>ke}oc_1j=_x&c8x!?Zb7~d8o^EuVG9lRIOg3V>lf9)fz6BWtK zY&8fmr*1z1vIva1PQFnX^bLrhaQ#=hcb_(oJpd;8&PgTdw_pcSNq@$`ux&)6opG3z zg8x#os`!fwH=mG8Pj9Q^I628-L}*ehWI6B43We;~Dx@i;ChLk&d&z>K2<}(Rsli0C zN}Qva=G!^f3Ufe2c1?bsxbz7u7srz&2yr4daZxxkDF}~Qn#lrmGN4&MNT{dF3SGR9 zmIg@(WCP;kP<3R+S+4#7I~qWTJ?tWYSi4cP!im&Zz$~Uxw4gbS3YK$K@Cb2ot?Fdx z230e_%GKMyke#{FyR+=C8vj5EZ(s31@go$&YS`I71a{Sf;8q1W*$3J^`u)Kh`+-9- zs)p_EZn3Ejh)cCAOsd1e1j(Jqiq}L*q{kKp+gxLLP2)-(ΞWLl-lyyA7 ztRI$bJTTLENY?3qv4-Ui56m0h%UHw0!2(h+1JW;pLt8ctYZ-tevgv*O0QgRa!gks% zuG1kgoeqQNGyu!#kT^~U!*JR!e$(Eun+}89v`fsUec?474y);qI8BGZXgVZ5(_yfg zf@SyvVloZJV|oBArbmjy^rZPbX+BSy&w#N8VjvAa1N*~1dcr;Op9jP|`ka;QKy0I7 zbia2@qep>f^iWtv8{-%q8p9}D2noe5+BB&y7=UFs_{w15l}8P_vNhn!@C)(~*hCM9 zG%^TgE)m&k(Y}XE^fDU?7K~h(DX+&m08l zXTK;vgV%N#y3eL7dr(xL2SW5YJeto2NInBmd^SMv8IInwH4M!V)SghpoJMr$H1loP9h98qb3u@jM&~&#qu!TA_V$MUL zNTZ^pNo($4wJ`{&v!MIpPXDEH2(@4Ugk&Kv@b>82^G)>Cq3Y*P7(wLS;W>*KzhkQ5 zq6emA}I2;Wi# z6*0|Y@|~k=>t9%kf_|D9OD<~XeLN>;HeL0D_)vU7-ivXTvl;({fF=q){!omq9zgo2 z6K+@=Dr3}FB*h8#@Um2e^n;4hQuP_9O2yh-k8ZTEgif6MaKt`Q{fkK?)3bCrj-XrT z9sF?gL7Zna@9P$qd_2jP%c^jTTx zYrEH|wzR32T2fij@n7xjhIG`g{+edSSwaPp-THW6Q=YJRi?M30c?ON>F4rfZ zd2QbclY-?b7c{|y05jCLGG;hPqu=aiMPa~PaJ67@ZC!&TZP1!$WezOc{aVg&XqB`r z7>j#N6eej~$e69oxtWmk#QM};J#Brs^wrW%K5HFrNeR03IxW~S80;sz$9;MwPyo5f zsZapf0v~AY*(4F+09VI7`5<2R`T`6ue4%xH_9>&N5l|2p^_7(B8`^R0F-XpRjL}Cx zsM4|8*hO|1P4MoVwA`^jADb)Y_@6yp|9A4?sGa?2&JM5bzf;_SJ$AUp2@cZNLDq7W zj&)ZJ*{6^y!g{@(yjU?B8uo|I3xFO{`eiL@IRFnzr+NrhEz6S+3 zDAaXWnCqYr*GC)Px>aavk7}z(k<;i_)hznG3Zres7~p?=X-uG+-Z_MR;#q!DY@Za{ zeJZw9HclpUO^=WsAB3!$(0p#QdOZ88+gy{NXLK57`F5X9V?~V*f~yKN59M=5WxtfN zpd%CHnD?_Wbco_+tsh{$hWh3LpF`*o=E^SQ<*cbuPwE=pI8C%JV7dUg17=JjnnWc= zDw>XmuCJGWg*cX6VJ;UWO*T$LizOd~d8uzz;6xjQ1tF_pRU2EEAiH77t0paZS$nLR z22ct6MSml6FDWg63sr!O9<%KhPy3ubPP7UVPoLz_CpmOwreSiZUEn7P@-vhmJqBD$ z6E^%U^oB5Js}wQI6F?I;Qq)ceObVo(oqA@gR;^P#Dsu>xdTGDW)2e)L#CN_;FcQd3 zcl|Z4&`lJ=r&x>*Hi##4y{~{P9ltXy?YMhYgIuz;wQ3C8sm5Xb*1e8AgB749(6T^N zAuQot0G?NH6mUXSah1y{yWt`D7AsS&{o28D=zre>2I@2j_kmb=(l`jant(o(r?hxM z^c6GK9R~bnP`I<68QPikkTzbNdK0x33N7;K)5bCIC!K(C?+)dbON4whLInRUZKV*tQB(UuPOAwm(GY$(UK|wSq;3YU&Vw1qBlxAZvopt^$V6<&9St-rBWIL3( z8e>>@oixzb*Qy-oCu=kZ+1ltlFpczPWSbbTQh*f(4HICOS#2E~uZ6j{1y}cU2&n*} z8f9l7@lK*&*WA_`gXs#8wBwh%CfHHjc z&!&Y53hN?xx#MxL3o-d1ptBjHVC>VKRey!FxIP+dUS3e$8$|j13o}Y+NRB2KD1De@ zgDT}~LT=;-@u5J#wZycq`kAwbq}v?A{zQ<^lw1 zT!EaCSqVHrpFLHkA)&g&75-eX=oZ9kYfE-LX|^<@Jn_pdOsPml^THk= zeDnRg@7Me4*t|j}*0a~M!t1HLnT(H>-@8K$NeA0Qj5z*Wa5qwNEC* z9@%?)hBSdy_FZASX36R(sVCx=|EEoU9s;fU?rS-4Opq7VIMN7bLno`vSZy$ zjdQI)lcK+K&nTcnvP(o+oicbBxU}71otBQ9ry=y&uhFF{;@d00zrmyTYHf+ z%YfXeANc@^wYX+;Q=WPp+?~$}`DSSW0NPap#368zI%bUK@R(hyj{HpKkpFrTOH}CY zcvPHD$p6Q3*%`_r*qj!;iW0M)#sVQ6^iBMXoIrDgrzwk3hCj6m5av#|(Smi=a8dmE zPo4DA<+F#OZA&CSn&KPYqM`h=LJhV;Q5UXUD~i|1ldCG9hUJltR{o_)jgp+r;Y8tC z+3-9C$em5b;6i3Hk@MwsmNOb_wXewGyl*TjVS7YDgAfyFTHI80x@j4rSL$ma?-4j0 z`m_Q~<3eUKR)nU~2tOG5Z%4g}xC61P*F89(q15@k-agt9D+l- z9|Y1}V2=ibUko#AKOoZRrWm8)hU6ez(fuHb4#5%~V44cY4;}7I-!o?DzR*GktX?QW z=1ocI=NHka$uOTZ$smdX=P;jM9SMTvpWYtkX`%FMI=)r_2YO!zci#YMlBOR zuQnFb3~IVbhqcF_IMxG{Gv^FmU5d^rw{~K4Lj2j_`arc*Bg4&PKIL%?Dj5zyA517e zyRwm*{%hJ&J1MXHvU0RLgojnN%Rn)-H~F1Qp+npTH`LWB+KtgG)!N$BRw|7(a}s3& z$wG)p2|5efd*DzTbLm_k^P;}NlYjfScH5W@7$H+uG=_`lk=Ix1Qe4Tnu6=ia2$B=~ zkEt_jowluDk0cOqKlERG8(exR{d9fH?ziVT(9BBJn3&CT28)Kr2#%Enrxomu>FtfV zGVb9H-V82_kD7Ad3q=dAW?ZlQe}5(4%ECG(K8}p-&@bJTX)Pa$%T(2bJIE4?4A)Hp zH6w&2dGwc8s=vLR=82rr#1_nHpV7_QyKdy>&yE4Wwae#_28K3=cH?bMer36|IeS&3 zD-FO|Nf@~2sUUB-c!i(0PvNybvY53tf>DvL#=lCzNR*VSVEM?A1^P=hAw0)>946wB zfIG`&*ABcjcu_^ZJp1zOdHaF7G%;xrRczF-GkQ$V+voNh(KC5&{`;l*?^oovj$e=~ za#j1U{!hm*#(Gn{C3-%pGt!6(bn?8F7p_nJ%Z91fW(}jJ zVhKYT&L=F3I2X%%x^#@f$|rMv=V~HmIg41_$tJ)*4djmj4Fz0z1zWQ=wPeba?`_@;1YsqtE(*~}#%7A)ywTF8FGjWX4 zvE^GM0#}@^nrXXb;Y!BN2v3Y00Dd7wj_PdW2>e7&IWY+3RZh4bt0nS`<$ANa?zwYe zAb7W!E#zK<8D$QxtUP?)z%h{y7ZiebjpI)WkBf8+-@r|m2{`Wh=P`I^EJw)6J;I0C zpoG+5aSAKa7^i?VibF4vUCyc#D0nim>222cGGz-2g@A+4g` zMxyv?-Luk3qb1HX`ppjV=FG}DZBO=&6T!^)$Tv;+ty|eda`vID=h?a`k z`i(JBdRwhp`_;);WK5o&Y7!TChG1)b>B9t-gAlyX^1WX7jw8zpY<@*P*zbKNWX#TI zGcI_sypjozwhlIWyVjz2zb)iFxo0>}*lWFVP>7-W(V70E*E9iK(_J7&99s1;>ph#0 zA2Lkcrg=v+>6rXO3Yot^Nm2mDw!1i5hi4d61p&Wut#$CxF}VcI{zYHwPXA95Gp_zl z+gTX+6Ws0P^Kf9?94(1;>!tn{R)=J;pjvL={IJsZWP zK#J+IFU%6UX(R-a`whe#?2g#|_Bu&jcWz^*R>`-uhh331QjTVivMkIfMu65cH%wE1 z#A5T}Oj?`Xs@;eUl{yxYYhw|Cexuuxzwhtb?shYBumsQb4W03h|hUf=4v*g}(|p%-9qSyMyb!D>w(A zpQ;uJqtOE6ow0+zW7U_VX&K$J!a7d!rS2#z3o@k%6_Gv>AlbBAj%L>0R7mUuTkISa zg1ixGge4q`7emQ~Ds=A{T-iU-^F+y!<~O$CHOm)vVg8N&CUeU%RwX_;N06Mg_O&iE zz|yVq*7m`&$s8QpE2MQ)FF~=8st`0~M#^~(!_zi47nto22Nm7-$F&<}#PoDMCStq1#c z-2*i}jGS0k0V!CLm^@IY)@n(v;?>b z00Un&kTn^?(3rSjNk(!Omr);Nit11W2s!sPaEPaOZ1bqL$46M{Lxf%Fd6>WV zSMWb)2k*6qU(TA&uA6J^6or_|yX?0x3$SQ^b?RBtzdHN!?D>gai!V?0s(7~WFFjkh z`=uFa2R4{z4Tmr)K+C(Ad2B+a&E~=kRAJO`p4v-#1P9Qa=61CatYKio=LPv9l{x#u zYvwS_yenX6Z0&N5-+6Lfh}DX&RBNP4So8`~G7+OV zi$q@gfe~+MW7cNV{o~Mj=S)D(N9>9jY8^z*Z0*?Vj74VGHx9V>hU&SnDM|GwSPq0x z^A_;g7KbaX=Cny*J#=?Pi^aAgkZ!|_)0(kOeyqNso%(Cwm?}C>F+P4m$eS*RXDy?c zCdu)ORw%w+(GTaF%Z$!}Y`2~EYI{|XKT3Y7IWalF+`(HGlMP_YWXGHf>93-3_8V?a zmxSiB6ny~>Z==4+cus+^Uze zD=YU%5Ed^`yI&uqb1T4T0rkMKf!6*_idpRC_vxRvx`z8LT^-N{^&}d-S}y2zx8hY;F|VES+<22^k3r2p6|hJnKc~?+LQ_Ts z1dEH<9RzO=26Tbji!8)H&{oXl${aul27)`uxcgEXw+r^(e5+-fR2IY4eyW4N($nx> zMn-|wjhC5$t+z#K9n;M9U7E6LNUfc$4%M-wwQ)9&#|@|5jodrTdWI`APB^gFqN}Z- z@cXV5Qz{S!#|cT5EK^8lmGE#JLoC1T`>5GMKljF;by)rdBPTLnp>Laq+rY>N3>hxK zs!0>ASNTTuf>ow#Idg|$V@9pY={^K1BRPTSP{~t{h>lgeY!$?-kB-O*ws5RSE@gXM zY3K@Ee9E@Y)dYlfE#qegSdMQ@yoKT^PiUTi{rVaw14=OycRZJ9qPe3vnhRVpXhr`0 ztnI5;Ep6y$=Zb1Q^-*tK?4WgpF=?A@ z=w>Uno+8w@#tV>&r4OMb?1i_94omOVMM71Uk)-FtG%VPyvpu=ExVSVgVCY1#uZCxCEHY`H!Dv#eQhO?dO*+PBhmcBxA zjA!bm2ex=f(iyg99i}s$FvP(CW>7CLf;+I3CAp{Qhz1taoiusFvlrw-sM1!}ZtA8< z&V+3E_8We;2M9Nb<%i(w~F>JG`==?F?#@P+vJEa1WJMB z+(N;t|5^$RgQLAvKQ|)G^#P&#KlD@_&vf63yOZ8ov&MTb{&rD)SNaSh9FvC+{( z^6@=Eg|AkpE2qu~#VOUd3XH@3bhCL`VitjkD0jN5?#JgEoqTEAPCSjYFP&paaS9Re zcy~AT$Jy*SjfQ-ZUmUM9TCm+}af(d|P$MY+d8)P0Q8W~N)H^PAC@lWyPAo0|Zz#dO z)K^wf2OHyGr$NCkLsoKrKM9zhccmFvcFvubmj$$)&~nYpxNraLDNRH)jumcjQwWBN z8E6?)&So|YSEk6>}^yOFfTvka#W z+<7MlpnO)db=&qc5ukEHqwf~g6eRU6Shp@6`DrN$4RY{V55_6~3$|#%g``R*Nm8IY zk|T8CMf0x#5X~nPVtFe=L)$)ld|U{rIc|K=?AKPefESZ}AF*3Xxw9mSE};1XC&k8e zYW0w6?1i19Vzy9Z&iUPJf6@ylMNa0tK4CIN=aw48{!Y4~YNY$p-)gSt&?s<)`_5~c`2vJTNh;XoFxvSudyodWr zE(h(?Kk@&fwBxgIWg)CBQOsfrDrW0K`+;(C*!e$Uxp|FB@0ia2y7~)W@b4~zwDuZ| zww+_gs<)&C7yr9d)74#-X=VVl5Y^UZhv{T*$r1@+wFnK~)RREx*2i4<3Nj`t-U*%T z-z9jfE0^$gzusQS#&hhHCW$ahs2g3s)uFwZ3Zo07l2?+q`T{S{QTm&<@tWS!&**ly zrxo23Vm$b0joFv4rMdb?fbTVdlek@@=g?l-87Hs-;JTc&(u1fox`=MkKWMTDvxoQE zFH|cWVCa$5RDD#*p>K5KS=b+{4NYU!8p`R03eg-9gfE2Y+vv?h;I_glhCfubR{6v) zP3z}y9Vb)3fN)HiSBo-NC8E)hs|En&>U{ktgyR=4+BJ+Tu2qdSuXoMfqb4#^mV7Qb z_B0p^4eEL(ICn;*L98`w17>`02O&ZiF2hi&ldz4Rf-xTv%<#d0XKP-QdC)uk%l>Sr zPW{1*JHBCMASGgJ$-+&~j8S<5l0t9}98+6!?}G_%ELs-OQHzHMX*w=%lvHdMPQ?MD zMr?`Pub(+i%M=-X~(D^>EgzZODfBQmT%P*JF) ziVts>JF1WysmIwXJ!{|V-BY&hX|d0WQPH12(n@JupP*k9c6_xC>G820ljh@zX~zt3 z&_1jZi#2b%ZZknrVyIEWL6|mEjykjx=YJB)Ilk%-9Gy>^y+@L%)44t`7<|7ScLNYn ztUUGsp@q5^;*EFOOv2kKe8LNuo(`VvN31C@yYroDhK{VV+;K}|hwVZi$@v819lG=T zBsRMVbJu;U7hG}gQ*Kn+P7uwiut1*;_gK$%QG&#nm7fXp%M9aKo`k0M%}knQ3ZYTe zZg?i=kTGd%E=uEpmBg_NXU(V02ukcPWQc070_4B}gJ=*y_Nz&P?5Z69D2h2{0R0az z1dItnPpX8eN1;*SmjPm}+EU+t2KmP8O4OO`59*xC5Q*ZmC z!H0zP<^$3IJ3ZYb$sMuCGw6?K%<0M5e-XommsC!;gLX58St*T5LfyK^FJ1L2_|qQ> zI=0bhkjOUCViE#G$jKm${vmGj$iJY8hy@RGJLGN5=J0xkjJ5o}=l)6<$x?;`h8CYO z_p5*1c7RLDbAO$GPCTIuXJLZ<-q$LB?5p+JNPZ7LLR-=AJij#=-P?`NZx&l|`9B|X zf0iEEXMs%uaf6s~0k$msK--u$5;Vz%8fx><4U$(fnGf#;jB5%RKBxrfZkzkrr0{V# zC$woO4PuRF_a{SEA}BtSWB-wwDl2T$L?A)uqVQE%mjOb0tAd5yvYEt?5IC9&a&XFu zDvl(DTgT!8efTik&Vp1I<+48u^*EU}3C9b{2ar{CvE9Np;X4eGR_N*x+Zfb~m-Z9&|-%suhiqukoA}e}+MUa=N5_<8y(KbO{R9 z+YN2|$j$48hj~PGg^cPxqiIE;4jK-Ll6SgD{NF4A^1KVWbV;lIzdoV#?@9EQb?#&XMJvEyXmjwhW;aBZAR{_Pl0Bp+aN_Xo!H9a=;y+uTz0#; zdrKT-?bd6>C)%ouQwQ`f+f3KMG7AXGaX0b`TE4Vk5|CD#7$Y}7!%*%16+Rs5Sd(fFO}_5gv}X?tecjJ#ZZcjHF! zY^uoT_hx9=I7OC8AULk1A71MOZpPH?M23#A)M%su0S=)&Ufb9KCfG<0R@OnMpt`b% zZJILl-P`{JiXxqRNbG$c-~BkU{jNQ+-PR0~1s9g!mG0+4Nk;!G6qJ%<6i7#AbVwjWkw#liQ5AGB zYuReh@Dfn8NbOqQ6sElmJ~=y8s|I{ONBvxQ>pIsHqJDE`3lWn|;a=N$)q0K-)^c=> z)9U{7mT>auq)GhTTglD3l$&`zIdwhP>J_8P49w=Q)0o%X9qmoxE~hIW_Jp4>$~1T>(ZHRL*EnIwxZoJMiv!-OHqA2 zHtTd@=q;6m(iD)j1`sdcO)6CbMO(ygi@SQ~S-)WKb&;o7bRb1v%2`9#SlqD7Fxd&3 zlfd|sROC>jfzGhkToIJuo*a=7{hJPOWs;}rH)UA{%k>7wwZbuCbZB|+Kti57R*LIa ziW7WoJS76X-RNY_2fM5X1GYTzB()mrPXeTG{rh6i=nFy#(kgN1ZMbw}<{Vk8kqoDd zTz_K-3jgPl5?!5xi)B~?gA_&mm`BJPzi}_LBq&gbr4Mm<*_#V#XBqkX=D|`=9(Jz+ z5-Xg^8hJ2RDU)sa*-Zj1$4Nvp+{c!7aj$>i7{H$?0^nqpkycEZ@ZxlsK4Y-F&fq|@ z=A(Yh3MT=0fXv4n6LXyNW#wFy=ax@2L$BD@LS+e;Uzc6#_-0`OepAA^9mE}%<@Al( zpwJ$=Y{LUM=bww+r88AwUd6KyFa<- zj)LSyVSrB#!#?!N@qOL5-k+~hQ1J8P{yyjW^1L(p`+C`bJfGXX#Pa9lV0DH(f5$CBUUv(igoX3=HozG%hVk zsn>QcQ2Oo+c1ExQ^+wgFCY?<44CgX>HD2YJ`jkoP_SsL^G(AGZTFX${kFOdbR_=CV zW2AvZCd%woji8+3nYdMNw6ECL$|83ZQE4@j!%{fiK^2E}`Wro4QoulQ?{(K-H8d`q z1fR9X_c877p1CRAw`2#z7|~IZ6hJpP0vp!DNozCaz=E9hXYIor`3N+ zitro^qN94HrR*DMvUPJFk}4sva0;A@mk4>@?b6{WtV*x|KmQ;T)&o&fKB4%O6p1Rj5f zvq91AALY3%`79!=a`L8#%Vt-?3`kN`s>CflPt{T?|`s zUXbN|A7e(zWbSs(a~3BrrLLl9F^AuB{LL}R{39gRnI4+$Ol$TcE@->WrzZandUk+G z8=`7xMk?0_|EW7qx9W9O^sL2RILkQZwDSvti8JemtCawY`S?KVM2J(v9qEpFZ79gN z<7!GmqC2P|slZJ!fu?i8pnOO)J$dyS$ncg&jx{f!db=1+CFe&+^sysai$1;{P#Ir3 zFgQ$yT$-keG_0l7&*l8L1Dm3G(e`nF9Cintxz?-2^3p?fgony&Rin{rsyTw16<{m; zy7~Ra!OQu5y*ujrXkqw`Ux4lzz|UXvH84wHhrYqDGycY9I6stR^R-rak2)X{&p>e` zDI|>*J>ELf|-*Oa&A>HS^qlnq)A&tYu3J(`BkdftwjYh00RuqS_rc80V zG}}&eg{tPa8!QFOEi5Azi0(PjxqN!Gf0bJ>u5XTTi+hez6O^PMh5FfN7VIx&& zfkD(aI|pj4*J9Q2he{2++ZffM8k~UN1cXBoiE?ttKrteq^>hrWP_^wxADc%XstRBpw(jvyaXsq+u!0^EG7jL&G~~i#6*VHv9DZH3VqHom88OmW=)-q*jCU= zvZEZMFnP{Me@h`iRGKP-fvGGLlO1n&h|BW6-V?a2PpFYIvPfQ3N32dMN!rnUQY0C7 zQKsm*?vYwZ6&6k?!Ax*CixUsJYLQnv;Wn#cp4ai5J#eOpXcl%YxjaR;D!0Zo=vM2^ z^WyyxOwJ{qDy|`5q$%%v9c1uJ)3&T~z(UAgPqZ5epA=g?TG(U@a<-q@B9i>(F| z83_bdCVGTBD*u7VZPUzmyZaj6oh(v93j({AMh6!|ia;-ZrQx6k^SeZdKKvqnJ6h-p z%%@MJ-|@u1^2x>Ur2@_E??q?GCYNb&g@N$>2&`EV@s%u|I1x$uu4*M;xr&ULyrs1{`&jB2US(f0mIME=)!avWk#&GE z6(3~UR3`tbK3+jVfhWEYa9H)qbfp@wPS=7zDo{Yyg5?7+t8XXneH1bV3@TA+ZF z@h28_8JkjUsOI^6=)DD=@xR_d5^bUvOp>jEot-FABvuYVe%0TsK|)n>JDXpsMaVi3 zFIY{BfXVCEc_eRjg?Cr^iRm{e>z|Gu%b8kwd!zmDo{++B)&ZURJ$IfNGacIbF_wzx ziQ)$ccK*LmCADQmze_DBDMKH}QH8BNH2pLvP^)UZfu`6AN>;fWwBO7YD;p30OtA@y zKpGzs5s?+Koq^};o2%6};`=ej)h_^o? z5!Vr=L{-Khrw6NvFso;Hl>-opHnvcWfFk8^uYZzKUJWv46rp773J!N;aUyaP0~h~e zH%WJ|v&f}-jKz3Y^ikoU4(`u-w2vJnFZlqDV}BJ&VOJkMb~1Z9q?yO2aI+L{7txMW zMIDukjOr~oDOA$5hrD+AHuacQTtk!!Ti?fxUj*Ga0vaRHxYxQ*;(lqM+T8=@;*q1p zqSwWNAr^oZnZ-1qR2?I)5SKQKM<4d=>~gJ8=JSf`HrkysH<$kBK_~t|hig;en3}~7 zvCt#bKeg5z(5T*>N??n3{^&ZJAoAN9DGYJk%{mi33tqDCXiVovUVHLq#yADL(#NA? zeT&G4wATV%^N?k=?e+d(;c}FCV>K z?DabA7OMaVH`y(^P4M9bJt6hKK{g`#vLgr=uN))TT2ozJr}5Hf=kn-tWBOzdOW%gW zectl~xsIbm`g7jgN_q+EnugoP_mxzSi!Uo{?~vKdxYh>78xJHg^Gfe%kj2i@$#A{k z{^$ns1J{;aTSvMhtm*-7r57U2F~-_dp@V58S zFS=e6WpfobmJ_Jm?RiD+ot})!>CM;~wq0r_C_L8mU1F(%*+x7rzTvA;O zx8jNcGcQ;^*Wv~1E}9_Cqk8UY!S2ccjN%GVPmw);zB=)R>q`PKCTR#LGanp$hSh6hkTUKVRn(~jj1jwI?CcN8*_bd?oXW)32_xJ9`c=W6NiOS&NtKu}h^YUQibLdmhv-T;TXSSb!weXmFV z4OrtI;PY1lf_&~gCJodX&3a`-3c<NSC6IH2&_r@+fP&{-BNxMozU9 zVA_)QH)}s1%zri(Nzam7MCL^?YD^Pw_NyhZ0fe+gGrpt==%auHxM$4~uWtX%`r8&j z^oEv?M^wC2RSOSr^c?!H_%+T>WW7k|Vpe^W%+Xvf-pIgQYdtx`#epAK4c80YN%;51 z8i^0Xxd=QHQVc;^ZsLJcAWT$HbfLnO=#ZmvE9xr*f>>=fWq<4i+W>f|wD&;c+geJY zpC7vl-=-mqTb(b?r6-g%W$>&RcW$kuY9bRif{1FGHS?oaW5I9n>o-riE+fvdpe>D} z>Lt5=!*d$$3@6JXgZ|cRfH6NBMy*PIX60xqz8t^G)&~>8`M=T^kYUltR}B5I0ysOs zP)`;J6!8Pu%XNu10|AxuwS3kJ7sm{)i~%)*Aj&&^U+vSJy21InTgg2_y5TTzEe_a` zm@?aVj#!6|wElJ3_Ihq|$ceJcquJN$!#~w)VmS^Tz_57@4EKhO&ddQ_WyzE;e_vVA zIj>t->rs*BpH(hxaAkA|7QNKqcW}(kTuB5-H&sl3kgW{D%i=UxaIqr5FA3wCSUK(2 zZ_laCv&df@5r#o&a0d}ItQYhtgX7G@!h;|`BQTT>IZaup5ZM>ML2KX>8^aPiDF3rSCPaAr6nR^zBM;i%sRWaSYhJ?ug2f(5FI`5 zdQX*~vQG{HC3qY|5~$|ZcLi!ZgVgK**SUBvB%s?}JEie>Zu!X^d0|qw>XpIr;2;>R z7akfM*^{xtv{;Zg5Ax1VmVrnny&Df(4wJB8)*l=QkA^Uh#nIS2k1*DURvA2D*vUxd?FuBye)^=y3ZsLZ-GxaVlT<9@TD<1Vtl*6STTzrVx47b9jNQs_Wy~{%P zS+roblC6wJmBGJy zNg2oW_cUYaA^GnMXVHo_a!rQb@{OlE6_;4+@3IZ_lk!d2^r474Inf>MKGp>DiuGbk zTd^{Amz7)C1t$d3j-lMzvjEX+&j$ra{nbr~ODW|~H~8|pdJNy(hRQT~ybJmFvBBUV zkZA-^OXd{(-nUbmQ_i?Q+0%$xOTCXSeF!{`$_o_!Ou#b&;Jn`dA#PW=nS&-u_&Z`gczLv9TL4%unT;~l>?fktZHejP0QsVOf4-JATBn0C%m(UCb9H>0*Ma2yDI7-S&f zem6RV$ijBU$Y!g@Y-Qf(hc{u#IOPhaolq(9m` zC+{84gRZ3RIV7cz_}`N1Ym;@)xD(_`9?;shj%`(39m_hOn;vg>!KqSb=QbN8TWg=R z{I1TBnG|Q66XZ&*PA0Y1*tv(E3jYlcRGsm(V%qU;U{J(48(X)R04*q$ld1~Frj2L} zY1e{F5Th;+dnB4~|FM4B!$JpWBUT0h@m+NA4hXu`CqAUjguUab6WQfL_BIE2<$NSN z=-`BPOErz)tl_I0>EWk$s1Q@7vP;MO2)yUzmfjO8IkOyE%j2d>_NQPo?m)8vQ}H8_ zlQ~NfaEY{FaF!^e@P^Ox!u*r%4z~uk)CSm=6YbV!5B%B+%P?4P4w*4eN6$1%Y*#G? zKkvi(Dv!DToy>XjVXvm<@USZGZGv@4*UZ`vbb_Lx=wnu$F2nRK4X!~GRXp^rIl_O# z`8m{dP@+F*$H#IC>RqR2^XfMTjl6Qwy zHMi3vPAc{_peUTSRC8Z-18X*Vt2>DutJX{>j^xNkQQN#?DMnQXq_m!@+gB?PgfCf_ z%WZ|m5ZU>$?=;i`ytKb;R2WhBhb~yR4Aa55%{iYTEf4%pzh?RbY_4(2Lr}XWf8e5A z-fupyNBu`_`Y5M3*$l&sG7PrUgF&qvHy0eWpXy`sIG>eO#cS3E2N0aey! zqW8{(xYEE1=Azt{de#c#weUtei4oskjX7Jy*`R~Xg$x=Izga3%VP9FY<6 zCTd7@h1C?8YH+(h*1?SxA!;KJNrpRXJ;tlu)jur;mE4)v0d#a5c6FZZH734l_@g9F zht}NAIhn^+S1?0UZXn|V%f+|#dn+N;{_`4C_p9R$%X8}C!DsM$6F~1S%Po$@kkQeh zGxdE^Hd{u3nOnDk6M2Z_$o2}mwW!J3X7jO5gJtwXFLc{Jf3;+sP|$~$P+TV|bl4ri z3(X5H4X;gX}@tVK$(l}4j zVo#VUXO3ct)AX6y2{L=4N>_6O2&y*T4!`=?h&WsUw&6Kg!Q%Bs!0gd<>ilZ)2b&Mx zW~9D>+S$r9eeP%{QZID4Tf$lx~f>K%N+aesqLqm z4Y&8$M;aliZnt16y7uBd#p!z-mQTppxi6c@_jQaHrcU9IWgW_>VpjCx(0 z-Sew^+mSSwXO)EB-ZFI4T1m7Gi{=p_3<+ZOCACKmD>WZzQ`;}hdIQjr@}JoR@-`7~ zDLnR~4V2p`#S2{1lYJLx;aI=^BpBGDy`zXMnBrCnEg61xokK$hMG-(Li}NYKS%S(d z9sJZTu=BZlvf>hFAx67KV2zvQH2q}j<$bX;;CAd)3q7sQk3l$y)`Z0z`bP4ykLQ!0 zHRci14Hp+*^kkc9TB}_MPq}T(FVP@9{Xv0)p*9a)^ylH!lxV^G`8YS%`R3O&{_g4X z`T2&%GcN@a2#8rDrgP-ZeR>$)7I@$~E2f0Rl_FRwK(7Dxzw7PYbIlBncHN$lmet7w zGH>vyK8W^sezRT5#>#zTu%2?RBCA@t{R-Y%a$6ow5aw8p6}glOsdkXm?e6~J-_7-P zyyvZIg0@%;T{ny9jB?w_Fn_vfT-({DSLL}5=<@dFog43Ahz(6^uAT~PgX;PKW}}JH zVJs6{R}ylytrR3EZd7N0rC%Gr)E` zz;ajbJ3|gz74w^~vG_`q+};*jTPvSfLP!UMnfz$h39$;Jb~+;{6c{JnORaakMq*41 zbrL6if&E;uGV5&%8{sT*$isKgslF=0A>_;O_X%izi&>S_6-iTG4{}bJID0K3N1!L# zgoLgJTM3dJRoI}3+aZp#;;9&vt3UiriRQc(MPbZL3a~@XspQd+M8v>ltk#WHw^+!> z8ck>C-afAQe;@gwlaiwT2M74+>g@av4WNXkO)T{11;+D$qe|?^SL^q#C4i;Y7q)0G zrC40`1;pArk(>^rZa|(NNJ~Oz8fp;810)H^GIAEPE(mW6wGpf!q&)`*tBvje7Xl#6 ze=~Dtr7dUs|9t-^`j4c|?{}3g`eLZ5fyG&y_)RGvBlm$Cqw%(G~wO%mj8N6!(? ztXO;8wa@Oy8pGC%%_7u4La0C!y4~po!NhGo$ z@;jlVr@dO2;NCeSQSc_Ilm;-a=Z9c^SCJRjv1b-=XA(0RTr4j>-WFv z5#znzngW~h;QC<6a9MRtJL#?8KG%VD^jsq~vAOplGS=5(c%gYzxaL0LhA?L);KpkA z?RuOVrg=3dlF>7D^}eAEO+D04`D#Hkr}B*#vW?w5#n8XaATQdC4$k)wUR6Zv7WNf- z+nlv;ZB`~SoXM=*#79EoAa@qdlP34cG?Tzg>WAheg*IBKts*5g$zjl6VYFD$=~QQB zqM5XY_d#)~A4St{V9z4Yu1t|qbYDUnZUSl?uwq68EVNiGps!t)bDbhY-;fiKvhMMV zTgrlLmjrY1CnD-*kQ;$~^|H3E871B!=se&!9R%{~Zi^r`YmiWum#5+YaxD?qE^9Tj zOr)e8bF+E&C@}vao4*|N0g33SbcbkUG6~0N-V0qSsQg>Aak18xpLKg32?9-k8JHjh z?g4iw=y7H7H6pYNDp)vZY_{J&PByNzx*08jKFnPZHzm!fp~|`lHVID3q&8)}JbByw z$9L?M!|gKWE8>cw?i|T!9M!{Jf@~+e%Z57&q9=5cjW(*u?wcQdkkwOOX&Y0TSN-JA z>8-o|kY)2ij;c2`V zlw2x2%x0W8o?;$?WKk;%g|m=xR)}o0(N3>b$xN7XobtFXDtD|-t&XYN~F!Md2CzQwXlD1J;(Rkul%F| z*tKAde>5gb7#F+oDlHlb;?t*CU_y65cw1_5L79V~1X+}_!LQdc6LThY=tWf+Jj1a9%q#{0X^|jr9||??3&H11_L~oA|2nabqCMI5V@s zobFaZ<;8;L&GJR-nYc2W^UTNw)I549^T(GrDUuwvA2dClK@h6M2gR7EP)iMugcXA4 zAYM%1%}?3@sOVF0PZ#LYiq$H0>c7(P)VZ=5CJKVsW*Z!WNX6=xpfDUEub?f&SUk-p z9Ehy?{iJ?A3(*;_*Ej9Oh-|2Cn)zuMoO*3QrunD+Nz7#5N*TKiYvH1(4#gB!*3kal z$1fcLYpfkrz-t%zcyi>U5JZ;vGwfxFf+2w$=^nx?!6yiGxqkLaAT18{j5XzU#>|vP zogVB==LoE)E$I%^Jn+_tOp8`U_a**hm$;$4b9<2fxXlI*g24PGF#2UcM!BHkUsZgF z$iPh>S~GhB9DxF#!0NIw3c^n|4X<-~%ci#ZLX~o6XLA>*7?MpNK{U8;R*q2KXPaFm z)mZl+^uBY`%(2AqJf0`i{fFYElH|t70acqTalRD&de!DMUnCmV3!T#f4t4drP*CzWIN7dD%!dE8wPy{dEGBI-|W(03|2#cV(fyAC`H|&G2jFF9) zNr+wq2$zGon}woZeW!LS>Yqw#XcI!>D^JCJ&B>het=yC@E71{gN%|A_jE#FlA??w# zeIeJmy4nb=&2GI`Of7DXVXKwr1b0wRVhyurN*W|P#CdB|Ik9c zX?6Trge_uxLp@#djMr&-M(B=v4z*icHxC{Ep{|#d)W;|KT%6PhX7i_yw3s>UZ29<7s1z7fsF6C_@3dz>_OC%+eF0TDCE}u#zUCsMxSP67LL>pQun{p(N zFgk~>gN)PNP6!e&RF?bRQhS!}4>ZAV(F0%xOW zNjy7vHNBiTU!d3S_Wvcx@4Y8rxxj)stbSs+wSb94&RNFUxwV;ndBIm}ff^iwpo(ar z1fkh{h$bsML5t3*K~Fes}a?HGMR>Py%xVCPne1Sst`Jzt^py( zet@l>MLF`rqs9f-`%q7A)$pj&D$C3ad4#ajE{20^Fg(o#&<*BTe?@$cPfETAc96zH=3XP}Ip_=P$ zM&_&ueVhTvjcp$TcAr9WZbCB>GbB0opu&Rpk9{+KJn(Q&il}r_CNa-kZf9GKG+5!L z#*F!^odh@&qkYP^?FJ%rM(IO;On)1M+S1pfcVh|D(01}Uz#A=C4~dOEx57f1h+n7< zIprVNB8$RaEgCd0t56F1Ps8HG5RxBb$az*?o1?i04^bsk+$Qjr%PmTwrW>>t%=(i! z$d|6*Abk1D+J22BI8Q-T1kBbY$r|eR(@gFv2RrUNmiQ5`kic(AbcKL%&Rv68j#z#X zMT>eC>JIjFbK;|pkqLVB3@bsDWw+G|n(@62n_TaYbH{{I+B4q3J8}k1^E)XHsC~+b zpbIOAW^?0{A{(GY-R@ZD_4{C!Fgaw)Al;4pjF?ufVu0}ArN>D5s`rB={*X{8O& z>mVIE7-ZeCk>}ye#iERIO!+3da13er`EwZpWg=`1(gz*AbZ)Veub6O^4OWmB4%QCi z0!@9d+N+IWDM2Nr+rN{?_^U^g2gxnVV_1qva~lI&g>B8(2FVt#!GE1KH}9x|eKNCl zMW-agI;O2p-Btl>7S<^);@yqFt3i%Qq?DEg?sBc~3n@l`d~_v3z1!ld>wOFFe5J12 z?wUt)b@W_>it$qW?9uMRdgUWrpft~60W~aDM|b&TxtH(~z`)Yc3xi%S%jq?3?#TZT z>DH$2^(sG=$zLO68WRayN+R8GWiW34)jyaydTlSn9`mx5DF%~$?foSEHPs%cOSj?h zuNz?9c~8(t+{%ehe|uH7V0xeGbXrm!jZ z><+}gv79n@zT4ss73-O9)l@uG4^!xfnH|M1^)o8X3xmNjnY?O-(_DAEJZe8^ zRdURHMk3B=4b~5)ntnK|EUCd6j0kjI%1!mHJ>9Owm?fn92{=g^04;V5Ja-3RBE;Z0 zU@v9ns0kA5=Ze_MN4gV5Ga!43J_vjBZhZJN1%!STh!xv6MIcWYD!;GwJ>SW{g<0sl z>0*NsznKvIWBcyQ<^b9>O1GdCCsd@XM^gt*;cA+j`@3=Olv5s)2rwqI-HU-l$ek2< zIlwpug%+LnZx)pU|FeZGNHR1Q(2F5x33lwkJEg?Z+i9!-Euo zHEuYiU6A_wOj-}$CaCgu8okX9XC+N?XIVr&hNS=Gcz3B~RS}N(9WvWqQuMRWT zRAMrEZ8uGt3~neJxj`=!`QNXUO*F|`P+T${q<$Add7sek*>jxwccXx*CryE)L9AP`AEI15f%Gj-$*Rkv>7ME{g;^%n=&^1nH=vu5yF5`?% zkyf=FOYM^nmVIwD+B*0*ZKU$_d|h~^Wslzag_!Q4nrX>CS7j<}`wg|>Q8GG}0<>^J zunivtQ4e(tACmwS zzk4HQ8dNw@7g%0OM*T2XoX=e$=hp{USJ7=B$-6rqYpJ`uR!jVCR?M`Rm-Gu*V%KLUU{!ulO(gLqN`UA=@$ z$iD_yyaYW}jFi5WSXPv{otHaoOZ_`AP(f(wHh+R}JG=KB;+Yif4dx0A+-nYu(s@-Z zPE{R1oRGy&`|91bEO>elvTv_9Fn4k_iew`8ZRjQXYR~7|#NLSGF4r4H><=){N2j!y zn>wNW*Ac0X?OeFW#Hrd?nt)f5k`f=xdMkI4ck)0%7H}N6%u#&aNII|gGjNuV?obQvyGC!(tHzYzxT-(sWW z_|b7s?x!I^&rV$uG%2;3pno;S>KX0r#|?NrzUg^^87lR2BDI@68u2lf-A;sU31NY&3AmK*{I{^8OMtAG_?n zASGrD-Bg-$-1L(*Nr8{b=(5&I8Nq%7C*8|Adn zFE0&Shf>%#8g=YTnZ3uq*3t)bH;YW6E-nqfibgZ?7(o>w52>+_gn>f2L5 z$EKv#hK%Oy-Ck)IXl;f6WO)gz#6n)cqdUcr*Cbw6+U{go7yz?u9ao0PgS(RHL)x~A z%c=tc1l6uL7+&k(xPe0JI5c$4XCU?>D^;54qDboFye1st{XUbnitn#oP|ss&%f-5} zW3Mmt;XLQ>L`^fVF~&XFwQoywP_e7L+CE;bBJQ|I$e}T#rck)lC`khA)mU;LgN{XE znwIR6T^jc}X>=t0ji(DREQPUPeT4b%o20`j7pV%^eLU?CE6UFGMWm@9boAB4tOGz~ z{yb)t!{s4y>yvz!6VN=^^YoZ5j%T4CmSqf!s~4XfF5dQ>IRea z;-gMjjln19$%DY)))0-tESmV;)Ej$&3|P66e(^Sq_MiCCcO^2>ydjP>6PVTI>1w zu1&i~|HfETUtypC2_g$=TF1=B!vVha+@0y|pNL|ELWvbw z2f8q<%lRM9xLdo@u1jQ`OYf|o zUvimWwzj{m*_ghrUXLI_OJFI1K>`_9ZhqhN20{WsOLqBn{FQ}$$|6S|hkQNtkwD=w z@^X+~=#N#?*nE&<=2At9?Oo+~YO^P7wI{fp&Q_ppe8E3ET-*1?_;u8~Ex9n$^VnDs z6e-;!6gXh!*O(BI96j8=mi&&==p;4Yxqz%*i$a*b1nM?avoO$H|FlIX+c>WgM&#bH zskVubsMwxf(l*JG?J$Vr*!-Fpa_POS{kLH()~o0V{kSu+{x(}#$&BZ~5qij9TLU3G zX6(*omv9w9Y29$Td~ee;L9k)Swq=@=rF}VZIY!*(IyzfLVi>l=WPaA*E)=Y~l~RF_ zH@|AD0lmea{Kw(@MAC6+g{}dv|8K)y^}-YwUGoJanmjW{4+>5_W)0V$a0S-k^Pn)& zAY@aC0;4>h$a(-vK(xQw=Hi6K@u0gFn-1$3{LVk;QFUF!d40CNHEd8wkZ=a9qNSNf z1!T}=z%}uaQ&6rnh~pwp7@z&1A93 zTBQVuzH}wl?~NX-`s%GIiE`0b7Y~;FB`(wtZf7FPnoW5)aWt-F)s6(A;8(8l%%egb zWYyRfPOwQ7@;L(TKC4Nl#bddnia-0~F_3RzCQ1Xf4%C~5Q{MJjY|6uq8GVEz{gW zp=oqRJzj>F4WMNy@_Z+3C`OTWjS}!DMeuxcwq+rZLABNgNSs3qU*^P`gJ(sc$;!Ww z_MO458W+*uH3l7hZPEu7adt^U)sSYlF@_$OG17{PYd{j4cuM>JfZuzSgWvOnolf3D zOrTQA%7nN5&o}lB$PH{Nl6dOb)R%X+*H)!~dA$5CDz41=XkYj=GV-a#Y9sg1U#eO!YJFf-ot0?$b*`dNXN+oQg?6@hk_NkVW|h z12AEqbRYG|ZY899n2kSWu@9X?$~xk2mvMbjveU#Qw)p8&Q2^ zCqdL!aT4OPJ7r03e;^WtA)qi~kk6uIK7u2aghX33U}wR$Or>v5i(r{&5nY}*-{GR9 z%4g*oY~vWHN}3NZbPkhgW1zdg6}MqnD0guIggZ<^h~{Q8dUg^xr*Jv*_wW2L&;<;{ z66px@*^QYw2A;N@{lt)Q4w8;Q%^;07ib6mzLRJv8meMRMR%Pybo{NN3yg;|{0Az+j zP77zJW-2WAmDu?h0VTLcUWb)iZ6fVV7!n=&HcjGBPsZw~mxWZRUp#OUb-|*vcGiV= zKBfeQq?j!66pSKlZH~1kM+Q2p$%QsWZpj0KZc9#b%TiyK-oCp4{k}@8+3rfL4Swew zV@4&6lsuftb&YZ4PK%^2hH*lL!>hQ6wx?_p%W_7c*=&4$goUMU^W=|A2?4UUWz2@}!9Rs^JsTfq^Evlr?9_V#-Vr02* zi6#KyC`mpBh39|q@lL_H23@ykR%|;fwr$(CZQHhOJ8x{;wv83rPELK_zjvL#&c(TS zMm;@dcUM=>i>@(xRue@EFw+nx!{U(ej3Ts6ydPWY^Lcolz=z0|^{hz#VMeu2ZrD5g z0Y1(0!g5@vABvjE@5QF+3l@v62zjOmfaq8LgQZO>v}vRea%Orepz?CHD;K z_*_6KQNHe_d2FQOvJ&P+jS18&d{>4e%}UZl%IIfG%G1mFefp|=ctoC4irdp@32E{v zOQ=SE-@w#JO)9KdQ{gPH31`!UTXLXnNX8hDNgXyb3q@*Iu(p5$#}Vc%)4Gx0k`JL3 zgM!wFue4^9Jg)=0BB~MzB1J;>AjsRClvH51fO>$XM^$PdP7Ll zzGP)x2H^t?Bh^4PITm?a#dBTigEpABHV^f#f?R*weSBSYE!dysu;f~ng;}rTwPYpU ztR@{D*+4Q#VwuI1Ymk|GRf+ItDo{&B&MJ1YIrs#9Z&tT^yoWj&$YOg%% zmTMydH$m%moEmN)rmE8Z21DVrs^JXSej(yu2n~Tn$7)E_JV2Ttqbegca2`xqg)R&( zxFh(|k|r3x+g^vLjcQTL?x+ZEqrL%;X^jLY8x2f;gow(GdCBJG7JH3#X7B?%Z*|f_ zS8`yzf8P*T{MmedR_;pdCl{I<5cBe9F_hq;u6IjHl8N0l_7Sxt!r4)X>~kjF#qt+{ z-cE3%BikD)Q@unryPlgwby52JphcIUBb962uYxTm)`4F9SQa86t>eqi7fitI?I z^fPClC;Vu!J1Pi@*qF(VvwfnwC_Hlyv%POFvYz=`<$brjEE@nS=A>Qj_{sZlzkHb9 zw&?Z0c)pJBtluv45#asauCML)zTLjJF8Ao*&)35B{hi;+!{91bjDDhHclVJ1j`{cV z=0LH36NkmjCBirobC8`Emy3iOz3-Z|9kH+f{3|1&0YG6RBJojR-x+s}!J`j?jd6BIZRUGt+ zkWSdeH^&BM_OQl|VSkM6L*Dgp{~1Xy4_j~%CTupjhMgWAR~wu1Uu?Jix z0F+fH)cBX4ahAC{Mp*x@E>Y1;I53H@`_uQnk1>9IPL*j&zMNNs6w+4)GdHg8^hwpba*b@*HzsO-{!APwyyJQtB`>m-5-a8VP% zfPObgJ0=h1!!y!l$@?HdFYDJ=A6;ymEn2t7-E)=QRce!&g5IF)XvNMzTvBlAdPeQc zH6J+_>$vY*xGM7WE)f|U+)O9A*jYm`S{I}(SVctO5B3N?XbT0K)A&+LzFQOPzadI=i&7)Npcv9Y z_YszyZo8^+ZD3kdI+2!s4j}NUW!pvzcrqs|5;(#TExzosL^PXma&0#~(2>I`@3R|; zKP>~kE|Vdwy8Q%vE|bB!RZ?s9e%VQBK}%Gw)O+E~j4mI^?nt>t9s0ENA(CXS{upZ% zWFw)oC>eMCY1_joGrPYsOD?LKE$NG_fcQg;;9Ccu`Xb#iWNr-uDk zhP}TAJi|`3LyYHZzHnweQsfOW7S@?&m`!;#=cGJnN1Eaf}OfjBkB|V!4ZE z-n-;fVtW%_L2l*596(S0qGe(N`PNf$jASSX&VmN|7v>aBNp9n%SMkYXuB>oO4w^Gy zvR@9&kM;)ON90e18SF~Xq{lY>^HQYvQY=sm z91G1dNh4aI7$~;V%#R&=fvzLS(`(j7VmnuGOy5r@d?h;lv^Ikc3xw&A|}ef2FiuBruDeX71{>5R3|Yi8A2MVA5_q5LsyT%;q~6cgZ$F!vvnELgsHwN2HghUxS_@^Lh_z7GZ|H)&O(QW^1C)#vTn z<9cjO#a$9X_RB1f>b8?wAF$T!vJ5s$85#6=GLRFHo^ur+MMzTM&}I?cGs;T9Y>_Be z_e4cP&L)xhr{Mj8**45^x`Ia8^RZQN{f>DIAb1#G&Q_H!+^3Mq&6;eP(SIElf&N1Z zJ9em16aBGAP-u)yTNLuwzxF3nw!vZ!XQ?w;1}RV)S-I9>YgA?vxtj^GFiBDNvl-2i ziVRZHAEvycA^7z5HA4IBI+AL$jMtR%XuqxdJ|*|a`zWAC5OLGk=}~KL-`8;GhalG;t@(7al@D7rrDoPG^z$6Q8z zCXf`!QYST#q0OV&+he-@1N7}T@L;j;reSLTJ6~jSzS`JGo}Xw zz3-knf$c^QG&>7Ob12t|5lu9jEaJlFWxkeR*oh5d|4@iFQ}S>lE5An|nQ@dwaW&in z4@INQS%h-79iiiLMJF4PB>S7SMyn%)Y z#a@CHsr@?F5KqV3(gp5tJ=f3A=KU=2w`;xi0%W*`;mnyV<(NpOo9pn)UIE}KQl`sR z89Rn#D8R^3oa%IK0mx~azQ~B3?~GHB@%02;S0^WMUDQ9ePVbm{?PNl?^sail(zq0a zC(nJ(IT`SN1SN0o>Ja*Ts<U%Nv%2^f>u&xU5u|43+EE-`5|GQ zuH~%Da1jFWqTM73)I((6YC~fvGRlUU%Huc=^qg^L|p?+!fnMw{$sDV5-chN)pyJy+-6SwblM^-=LfaXz}J;Kj)Gm#dqfp5}g&h)6abj)zP`kK6l; zCy9JRX)=fodw4%ns3b=Cj|S_@L;TK#>rd;;f%|{nO<~9$R@?5To!<}%SBy*SCiwQ- ze!m>Y!uiuv0+AER@7rWg$$>?JO^z(?j9=?m5hadZ&|Jz~v4ijI{!JZz98Z;u z(+Mn<#a68$vZ7cRLKAI1yO(gN4<9MGWU%->iA5tAmG#oRy#G=8jsZ=y`la^U@%?Vg zJ_@s4zgzX~lZJSS4LT)IUwnVr?xXE@vj-i$tO{}`^lCB{)l7U;-*TvXq_OJFZam2g zo%u|uex}jt^7MSvOQrXPnR%gR?UHNQc%GW!vl-Zcpylk6T-ex2&HZi%=W6avweYoCc&QNU^22iJ6FTdn4*hXRG35qZSfh)?t57XaGt&fa z`8ZOna-A%FrO){3B2(4VwK%A*l6%rJI>wH>(K52Z_^oCEL`p)gTX{2r+O)JF)}sNp zZleRY;#u~iQ#lRl8pLf~^P4W-!N@<|c3k&R7`S#*PhQ>D)zK@IKB#gD%Uy-_$c(v<1##U| zb2*|P{sy!bJsYAN-trJ=H8{zdhaBTv#=v2Qj3wQz$WS3kgT)GJ z>XacY(m@!H|7oe=p<0X?`U_TRorJs~gLjNtzH-GQO{qz9n#n~${YR(9;It?c&39T- zTk6=mSdmh<8=oCt2EWj7Ke+58leAsCtXbv@j1Bq`1r^Y;!6|McC5yX7J$TBps5zOH z>I$9UTG*tgU6xaG00X&ZkJY1RUE6Rxo%eY$^<5j1hxQAUHTmZVq$iA?P+9dmj!fv% zsORS4Ih$HV(cBFpnUy9(ep*dd_9@IUmr_GvB6t15*{P8Bf}xS~z@;0=?9ViHjR+G} zU`wMu;ZCL--;K-DXN#Q=1;{Qf+x!fAZ^$laL-lJ~g{sY2;FijK^#V1* ztIF?{oiN=*vttk~q@w<*)zq_o^-z>Tzpx{0X3^bd?i+(o(wd+K)(~=;g{Pu!A|Afb!31H0xWv`Ms*q89!upUcVZ&Il>{TJJ z=C12%i|gvAxU!&9*U;HK3Ry|b%PsYq=ijK=OzF7dtwxXS)L|&e$aNmr4yZdTS)nm{ zL_TDR^>id0_Lu!t*JNdy1=KmH{ee%KLZnnv*Dj$YnI$43)rrNK=ucjB3RCbAVy;?1 zW4Xj1IZ#b{`PZ+FyQw}l2r0LnxK&@<7wB0HA6>7oH5;ebmtC5HiZUh2N$O4DqNF30 z7^qVcy2{%Xff1ql2qBMPln>o|FcZK8O>tFkV*B(_J!}mlQQ`9_3w<`clAo}7b6D<^ z2Gi-MQwPH&BtEA5iU@DhL1$_FKczR9otWi`zG6w>E3J|AJG4q3{?1^!yQ^ZvD{4ZMEvQ8947X;5> zS2(AipZSa_v1e_zcA}RLBOfYf&!gwZ&4ZQ4)~6eTPk#28gWsR(bk8<|2e$weZ@OQU zoj(%EG!(BHt#1wWNZV)%g|hf$YD}z`gl=~xj7&QE7&mI3rQ@D!7zThpC_DFWntDJE zc{0K@)$_!w8)m2Jq2sQ3z=)G#Zqtp@YCh8@)v;HR(DjNs$ zD+h@rV@WdHlXH&lI2tZ0|H~Ccu&U^E2SW90 zqEz9J!6cWd(BtDEznw5(@KDMM(<91JbfTApj&Sk#RIk6E&gbhWwfsWPJr-)*AYC|D z)uRjn3OS9O5eD4AsW2~pmG_FN;>^r5wgWkAJ%5?JJ%u7!T+@WDR0oZ8dHu-8EFfoM zt;oGbcJ^5jB%mIWR{Bc{H-e~rAs>OE;TB|pg^_z{(y6@O*^AElQ}}xe#|Dx~JH?@J zE5wqQ#Z0-ImRBYx`UxqIzZ&WJ8zouwf#LwhQQcob37gRGp8KnZbS~it;DnKgr~kJf z`X}HXougNWBpGHI5>d$3M#aEGly)hysH;{*k$#-BygDd+Ya#)={im`BjU}19A=BE> zpSFY$H)Lj%paHm3uZ&sUfDGA+_<=;N+(oWkBuB(1X;B|$eUMCHjg!WeD0rES35d<` zf6UPW;yGmqy`DE!W20OJ%Uiu+vnHNWCr8+|h&U(GRNOHWxg=vja**j_%2bjhPVzC1 zVvr1p%Ez+l!?})*nxe3en)*P8Ja3zLu1m4d{c*KpI*sz!sWtVr@i>v&6h@+yL+hoV zij|Uc%<2LaRJLd)`^FeeiC#@Yk^S!I_na}v%WBT~!hxL2v` zD9j4kyG>e*OS-=9L0&DE_4=00Lz_eH<(|k$g&ah#HH>Wr#S6nsNAwXD5gWO;#u8;k z1a*uC%^77%r>%jyyv|#qgShD#z>cygrXzCIb-!%VYLb7FR$?czh*+m2ZEx+xN)`#T zYs}Oii{VQox1}0(>y@mU%bE}|+g=*quM{3s@atYlK0V2w&EnoX350{-C!k&N@9((Q zYsV)>pIE$_^u)2!vlkkzrSghzD|b&6*H5ID%hRkD=iJJ+-XEbm4-ndQ>TuGUA}gSp z6(P~G6-XGS0)WC|>LB8bAC>o>fnYK3bK;?Dy>Yc&1A4Jk(zQM0Nyr+p2*Q;9H5CTw z>DmIBuQC7XI;?`qz#yahw~Q#v*-S9#2Wjz|6OJtTl;!E#KeYdPMR4Kkw_n>D1U*#P zoAFv!L}0ywZT;+^y@Ypg(zJJQQ#JH^2{41%iqYgZZ312gexHi{-+x{qw(%YOOTo7H zKs~M7`(!zQ12407lA0;!8*mDB(AJ$tooF!~gN#A8(Vtv)Y;857US8~|vH@|q3>Ezr ze+g2kOJp>xG0H!peEVFM}<&h#U$9c5IwmX)$$Zt6N2277GxmLBnH;CDr~;r zToT{gWxZpfrDVneW7`JXVMI)K)!?N7HVkF_;HOUDHNg$G%1|W zzc<}i_hn%5{BnRJIxt;tPvg14(cM2LTU2JPXjFEzirZQ_ z?JYEp)`Ew%O2fLM5!_PmH;aQiqM=<=Q0^GWH*|#Cx;*V& zHV*Cs`*OO*;)w?O+fCnt{aX}JSIV&MT}A5FfA)1#ex7MRwmLfSLj8Q%J<3OX*QhVM z{1_*0r~-9Geb(sqE>#QKynXOQJ$s@tZ(0#=>7?&!Rejos*;eGf{tnq+@#beQ3nTCTFq!Td`%ZFP4% z|5Ez@N_1C9B1;=*0zn`lkUM<(z3RlPd6TvW$@WJ7V*MJk2=*+lZwYg9`#Px7U<#S2 zf&;&6?V8q{*%^yAeSu$oC;YaHZ=COkjoe<=b)uYw><$Rh`e`>(5l^_d|J6H^tuLys(yAaD4UDRphxyZ8kp%d>m5=NW*GdVBnBM z7ad%A0bsG0_acW(9Ct$vSqrF)7tRn^qgs?=q|w&TV_!%7b)Q~UT0S*)DaMf1ZYU%m zkppe=DlI-fMc$)2iA;_djxwmN0;5l&&|yvMs?;5^N28nf3kQQdI2%q!w^Jr&l>5Yt zoM?}u z{|(B8z0#X{7)AZ1LNV{(OQXfA0pg81d!k^tQkEDS0$0-$LV;_ClRrs&8?@rcM;ejQ zvcz$wNE|e$y@SiLXD>n==E$8#B?t2wbcFS%f;HsT6BMwkBk_Mc2+lFZm!h9V0N;lI znIGTm-xTKvr8z2Vb^vWbIjbuvMkt21TGm{io}B;jZ&>J!e5%P-pE>&-5yN+ZX5zg{ zgXfpSEV0XdEk!uo^y~-NW`pPPg{8id((3*Jw0eSN@&BXwAhCKsS?HLg4Jo6 z;SN}7Z52@4>IBxBdff!ko_f_HVwpA2|fM;VD{e(*sB3@V`M~Y#$f# zpY(UbA$eMDAAwA^RuM#6XFhE{(P>|x68=?JHd=MT>9$%8ftqX)!Ww$I$GhUg z&&mmVsh}S*E=I6pKgyyGj)5n6mkX!zt%egZNdYT)%PD)hws!CTe!x;~9|vQ5o}s0g z9mUNi@zHUP)x4NP&Ly}nFQXPYXr*PV(s?zo70&uc_maSX?kWG>)^!Q;A(?cls>#KU z-Vu8F5-@1c2Dsh7Zo2IR?2x-YV_#|^#OSUjUGwtNC!M3h*R|tqCyn&Zdzasgq?i?a zw@%vi!7X2t6AZQD_@c}%5kJLAK95bTO0<)M#jHhYZFd-rI$Ok}Hr%O5 z^RlxsqyxNYhBu{wRMy2$E>N5laC^pTU%OsrGgzT=4`g)Te^pbE?WTSUVU_8^&QPU{ z$V(okO^h7-UEDGRF<1JWxyGOvRZo>fkY&dtF09jBdw;C!H6@ zc0D`YRXBK55(Ecm0ti|l=3Z9K1H2e25pYq?z4UDkO%4YE9$A?ASW|vqCWk0Qo;L83C&+NSzGT+DE1Izd_-?sMur9J^WL4ZEHum5rb%ke<~ z{)~@c!0&zLhv4}tF>jWtwp^wK++kA<3cJTl8}pia+7jL>_q; z#^}sUXevMxt>c8c)}GSnyJ)Ew(L|PotT7ZLzsRx-X~)0!m_O_k#T<8_N_aGPH19S||6h#`<gt0)Ivw+EDOrFBvP5V-%UkuAn<{*1w=m5p4KeBVeN@8F11$m{<1lc>g&{&cz27_lG zXoZH0X8}f~Qz@@9us=Bj`dxTuiG5juSTo5vr&><O|6^BB1@Q8;sVde3kh6dLH<-hwMJ{|AN@;BZ)FChKhVe@Mol_cQw_w&@=zN zk-Kij_M*=7b~Qs9WjF+N82O)@<0{}>m4OkBq|Vrn@*!`>;5=Bh+osDPWPR>dYUTA5`)U)XTL3CiR2l@&D~X*f)nT=6E4Zr6Y`x zNUs~IAyr<8YC!C%a}>EL%E7H}`?A07ZrdSezoRiJCt`@#@!<%=lNjWrZM0JxV1x>} zdLmPMF_P;H{>UnQSpU&Z`LmD&3B?{>z8-=nSQJV+&Rk7beSvZe0IB0cl8-H5DO^+< zh?p_+lsZ1dgv7=@PI7eoo6IR02bK%V5MQQ}Byp1S_b3M5fC%8`nm3Za6-;UMHjj3L zb|x8rt%y2GQtH%rn$l&UN|;_(S35-w%w5VpB5AVR%x}eN$ra`;mRrQOD!jJR65+$5 z+!#;Uu9<#xUQH2_VSZs$2g;fIf2y`dqJ*FRm-mqVtLf({`ciKbal5KUt%?m7RT@Oh zG~ztj{_TrJ51X5+6|_aA$1TkvwSMLi7E0o?bF89_x6gv70vT>M{I*3N?bd!Fy+`B@KysPmY11%tKX#>5GbYKGM2=5H*D{J{T z88sG#2Z;?Y4n_BP0&uGaqI2s8+qvv9VX0R|1*(Mza2`u}RG4R`LG2kb5}>`QKJVvb z8sW*k%a>T-D@mm$ld}IaZ&~S@n3a>ZRB>)pa{K|6o4kBIrw|{eDytX*##k~m3#&#@ zm?#3_6`L#WHJ^|o%4{7ZvE&Z-#iSgMN`i%785y8Obz~!F%t2_$D7`h2Xvjp>J%THH zG3Ys#LB{rtf1oqKURsXKmbonI`hhmW-GVY2Rem6T@hLKuEIQ!P8h9`I+CFyTfcuT!7^nft;h zI`-Y}qu>C#0vO*Ku2Rn1e44~#B}ExzePAD6tszxJSA{aH% zPHe{LeTBoaFIw$s4lwHNB*zOEOJ@<&NLH(v(3%`trZi}1HPJA$w={x8;m+btFvLwJ zHj!IQ72(lF$ZOSZssTiS3P|jZxr}+~Y(~QB!iv_D;|LU==m(oY$7(pC<4Jh9ow55I zY+N>SXQ`X*l~WSJXHH5bGFLFhq`_u`aY|v97Vzqj<(gH{FCNk*W9#x$51LpNo3eTK zh}wnWjzG_*hL7xg%O(&}Hk3L#=dygtrg$vpHL_{kZ2Hekx;D#JAB*JUVit9`zfW$= z#~);qYQiR6NR2ftUb*)Tsm$lbF2lwQqRac0mB~@X?VSb<>-2n}aLPHt*b;fsR+uk06xOGtyxvY<*Az-r^BJC;7*n}Sxusd9qS{5716tv~IdLsEdVZ@H$-o?K>xkJ9Vy-|GfB#v? zHG`}CV_w<8EldWWibud}FA)Y90gEe}Q!opWFdTy2yTRvXN4)e?!D;W~nC>YCLZ-zl zU$_1xBq}~I)Rux0MR1lx48w{?my$bKr(orrA}8xpG6J`z%rEO92pn_lS~|$n8|Ixl zt%_@I6|PI0U@|cQs_jr#K&ClG`1y4siBQ~!A2#gBRp!dv+!@w!E?ZHq7vzfI;~Oqd z=L!uEdXMh zTV8#(1;pg(0i!{i`$^V44eO5K%Hd2;jr!zBDJ_i1$1S7*iLRJ{a^3042Wm2zS&h$rXWBgj>_wMSiK&?KANE=_c zAp4vbL1#@_<3|182)pm7ptIE`u|F4@1R`zEaY&1q_ijqWBJ6cigyJhqglF7<_!FGE zB*M;`hRL5}VHa)xSb2AC0g<+N1+kLiqZ7G4kb|38Mvb%hPo1GgoOux=?YIF3P4Ur9 zBU>PY@<w4z1DEwI`{z4W8EIUXK~&wUXPt-1qVMfWbV4233}IPw4n5ti*~I@VY9G zGG-@}98n`#pqRBrR;7`M(ou_oS)(mYZR=&Efgntr-94H zjxwxpkR0r_OJQE&2E?T4O&jH?v8$;R@tY{8Z%w3?bYBAYGq?=q|`Atu=Px-V#+rR*2dM{21^IEDZaZ8j20 zPFlm8z+bkv$Z(HLM0ya~;VBmP1vGl*5utpsY{?QzpHV?v=qBF7Gb~q%nT0@9uw%k{ zw4xT*>|u-*&K4y&TD(hvHx4q!Nl5%Nkg@rUGRK*iE2?Io7kFm?d`o)EBx{u_!@JYG zG&zpK)k$rzo3ZT01-^ay>fw&d55C=Nc<%){$)Ra zMLvlb&hWe&tqm@X)fDmS0;Idn%@B7BTT5@*lt2r`ctxQ;X_v>nrnhqRZ9`Y8$R=-8 ze#a|{7-xL+B*s=V9gO}JbF}P(ra4Ux=6S`s@VG_In6+9+bz}vGq>^HIRmr)~RdmsH zS=XYN`1Iwx1ZF$*2j=~vf4^UK*#M#8!B^G5;>eXfiipZ(ut}GCWPVH~rp3*U>t&Vd z`O2LZ7gb9fA#|1B@AABO-pxg+qD>F_c~Q3T=Yw)tUp#bGYnYY>-+|k{|98y&d7EVC zoAMaj+(uXBcv`lJfH{Yf<+(3vDPZf%pJFHlhLizwrRj5;!r?CiZ|5w**y4Y5X5?XD zvwty?;|fLE`!ul&yJ=8HqX3!Y(k`v$6|U#V>DCERU(ch9{G$prWVJ!RMi$LH9?w>l zx$I{{j@=_#tjCWZe%G~FnLani56`kE9&zGxB6BD zL8LuYhG52~f}n9x@*bQmGD}Zjha_Y=gIl)w3&DEKgkG*79>-;W6)yNn2Mf7d0kg;X zUZHe>&S>sW8PUlu0nI#|LpLQ?00M&j7wYeblO-$m)Bzug=)%nW@PkxqO_F*!8%SLj z(5KdrcYv}a0u1jE;(%YU4@#4Z{zalDkI|nutkq~$x*2_~%&NB45@6=cCDh74V#6>m zEh|zu0ORR5iBpiY>c0;W8v+pz6x1?G8`6F7@+bpwX@;Vry|%9G1jU+QEJX>kGKlhc z*cuE&31l0n+?&F|^PohOc!5MsUZQi#;>L<3<~ar%m2oXK5LGM5Z@ksn@Ms7jcx1W%o`JN>FIa zZ_Gi;fiKmCe)=i)EM=EcII)P6K@ zLrBGE?@uMeZ4Oij(eUsV%>a{iE6)W-K25x0je!_e;x^0H>F?z3szdU%NB9E z%QK0qtxO5Lw&I+Nh##jZ{cx+KTk|yXBr}uix`pmU3sjK}kw8_4u&nTZWwp69J3Y!~ z>qvBfJm-;=3+&cB=c{XlATn3dCL3+5QmypRLhq!hTLmDZWXNHRL23!+pndJZYFxdY zjox*1o7)Y5;D-+iO)Jwg)79Fu8Q8AZd+3E5hi#e%YYjzyZWLDhU7Cn$_Mcg)FT~Qc zH5&5;Ee+>rjy5gBXF7~kV zZsg{TNab(Y4N!W8w>XbI#3M^MP9AmQ1|6a2($z7_oZftc_8 zgs^zxRY$sQhOm5^2 zDk78hMxn|MnItQ0oWNEVEo_t&hk5U{(i(qVSrqE6OSq-}_xWH+xVNUph49|1F#f8# zB-C3MhW_9i>Z6a-jYpN8a-BT4g`f1o-a3}l%&)?BZqt`M?O512K|GvL*eI#(O(Vgi z!uD=lIH|BfqQVB1DoaJH!4*_m3;>gcXf`-^(5iP~X)`!Kdo-+fX>K(*XZUaUDBk!S zGF4W{6lKYeTq^zlj!o09uu<9)GEEMff@w`hD2=r#%zrzVy^P6x<6Gq{x5~4y z>sL5FK-TM2I5uwMffT{8%4uzqN8Jeh-`O``ThafMr2l4Wx9X_C_w}31VlViG*J3YQ zd%icF_P-jKZno&lZ;TvCE%U!jOEA2iC=)3MRiEBRuhl# zDWe3NQx*}lw67CL`6q<@gRHn=LL3Q*fW|f91THiiFpF9abnnVir*jbugXv#0FQ-t_ z+0UrN(E-u)Wg^ND_D$AKDq@s{97M*QW8^>a^rDd%jzk?hmc&$Ys|FV!4l0f~2JqS_U|%DIVbT0@=VM%|fobiD}GBJi9oD z*gaY4R*iIW3dItIeeSlg5VBvc_D6C=N>WkUB?D1Kmo{`M#sQs(sMkkQOYPCQY7OKJ za?(No!&JX0d~+|Zc8D?fc{R)JY)Ba*GIQ3Xg`cN_w=kMF5enHmFN`e5K@LWdEUXK4 zkm7<<$j8*Gby!Ooe0xQ{ynL1KPFQv;o3GKprqB~wmU~@v##|~EEqYYbE#{Dj(j)+c zO^95?I68ZVc+eGI!KJy1pi-k>HU%rOvMdL~s>8_9kjtY8TT~sFp{>h+w7>gKux%ja z%-I!sL<{+3QgvL%G}&wV{C+RKrCqSc$~;u*#yR9=Ix-526!?*9{|ynlNH4uX9<%&( zf}CiBEAkk>Aj_6c4^JJDieNWM{1vB(l^`2w5PEQyx{15R>?=JtDL^+YfGXn%^Y;GY z{5oJwSI;p$6LaO5v_wYvS^V39V_T*2n%zV~=)*ur8HM%7*;ER(i98xfz9Q$O*l^fv zb{`^0jzyKBJ|2L`i8MC%`_a~Ca}3-dQ^n-9A4)BIxpHLyvx3XeF7Mc-1hksrSqL_C z;WceCtRdvk^xc!)X1!+XdGCBFaSAV`&N7+YTcIA9O2`2&%UxY3)FCXLQ5Pc{^2X(e zeZ1v93(sD&OS5)=Ky{3$*e~>l**Z_v5|*Kk4Qc^-LaOK1cOa5ov#i_qNKX5*TnNo& zlxkLZ&ciGTFQ6Bs*>XtrVYMH+ttm6`@&Rdcc=7UnLiTCOi&oYU*Kx`dHWDD7g3}<% zx=@CT9Yc|KP>gCX>TNJpPz-xhZsp*82@|jN@S;zR*Kbb&#c1X*T@r{H+JPECrj36@ zCZ(fDed~xjf4js$nzOj?m}xu0OLXcOv0_zUO8&@m7ii_4Ixovsa0ZaQW2j~=_F!(O zC%;`(PiwHHNkcs zufprFee+ztbf7A>6rCqCQda$|dc>>K!DQi5JJuwxh0|EMtxW+!r~ z1zWy7tm7VKREBC3(nWi}J;dn3V%ox@lBq))QPgK)t=CAADLd1aQO1JC8Y40LV|EtW zDfp-~jm5fzD=KUcY8)$ov%)|2X>4ZkXuW>CcIy?n^um&^NjFL3-oV7&l|?kN@TBj1 zFp@~%d%*3qnuEcy&Th(D6VtnKjV@lZptZfT4>_E^NdYXE+MaGzSiI!M_Kuu8Hw6i18_e@p#F%S1X>X zQMZRdK{gfpLSqGlgV!li;t=4?P*CYbK)gZ0cX`$A-FW7}-+D)Y(TgSumZ-ZdUIQRe zc@c=h9fw#J19V5~>SIQFc5KtmGRihVEyQDZg+EPLsvb!>2Y4^64A650?^qFoc;LOC z!}n!hhW5a0e$?z9Z!p4iEJD(M+du&+^MmnWV%%!|_N0C}rAxOy!~6>Fz0fl|jSjP( z7)!TqQ~5a?lvkfdG#)H(`4*E|hpebMyF#Gp^mXFb+Bp#)Ym(#^xA}THN+T$+ydU0j zIeErrY>mu0Fy0kf>rw834xj!JqG6~N8x7iagGYin5mmgTY_cW*RV;#5S8ggKjkVSE zSS%j=Mm>=&fC_e@r4z$wVS&eL>(MO&FXAW0fPvN~ECYd~vP~;Nd4!DW#g!=h;;{6@ z_s>);u|)kLBrDc17i5V2$KEJWxxG~VI_O% zD`-Ne_Zq-k@)Zw|`4bkcmkR*0csATIZ4Qk_pPaM8>a43nbfTI^Z1Q~2K=*(WJr#Q4 zW@wVz%kfQ(oA5OzLz~EHoURQ{LKe}TfP$reF`3^GYx&49<{x0i>P?3k94`SV6U(O^ z?G2K(C7|0=rbxns^avXLMQ?`H_~+n8o(f-CBYQUKY#M!Nx+XVAF&}ARrmw*+`^m)B zwS(bK_sq%n9i30#BE*D(P;w`_E>x&^P(6a)&F zmnP-d+fx%JT9_eZpM9Q~_4fLdNEv&AW6~}BSr}PaiFJACcl;2jX@C8&LdRq`xJ3}u z91m6jXDm{#;%xYk_BaFoaWQQ<6X=7*Y z?VAIjVh-Im$hwrvX%*DlhrT6*f+6nWbwX63O`Xed=`>V5FFl#qz6qUtB~=JSEW= zEFq)@)M88Hf7>VWSL{U$X+JL7Ucw?MX17Ax=`)pU@N$&_tdA?@j3*TXk5N|YvzkyU z@rd;{K#VLIGqW=HP~@!7pu%2_bg5%}0&K^Lb=9O!puJ$hGf7}Zg4sJsOE^p@Bse9|!jBZyla)khfU_2KoI<}bi$QT9jF=*Hork+Z9I%is zAx^{hx0%lt+1SacYPc_$I|YbkZ)y9H9QB28mT{AmlBFnwP?lyZ#iZrinnm|b2r|ho zWIQVM{pt1&Ji71%V!-m7uwMJ=Xep2XiG7vT_}(VU{qCUGv~?3Cy8!~KH8j;mR@%gv zc?ds%vbubVWPx`PEE`?PLgL~@HShhugE>K$dX6(z& z6$twi3jhS2WjUpmtOsIPn5Xl2cosjU`xaPwXhcfH6*KvN>_UYtUJ)^=- zQ74vme;&y(D99b~no1*_??8P*zm3b91)n3aiHQ2svhq8i4C5uee~{(~G4s3lIpdz! zrNT@y%Lvah%FN>BB{*bfv!PAGbt}3hF0X+n*-4%H1%<6#1sTneCl7+dO{a)Ae-v*_ z*+o)vkjSELW+8JX8`Sr%j)V3~c$3sOBaLJ@z!{U>cQ==erV`D)`Wu}+XZCzuG8%#elh$2@&b!UNbsOo^?y^xOP7rlYC z!KfF}T{?Z;eBOTZsb_X{z+sRW#EU>-6m8%C^b?R|*+uehV_2-T84RhWE2f5AcvYZj zvfuo+_M98P)hl_(v2Suz)a2H`Slc!P15$Qh=0)TeZjD=8O$)cWs>1$3h~70{v|ti7 z!=Kjc`d#Jw>g{5pYV}4yZPAD-EBT5~y$4uX0+~rRQJXM9G=8GzdzUUpowU`N*0P;q zF||n`g+dNDFMbEV3GmDxKl z<slitLoLKdmdD!*ZDRGB+ z!N~Q4vDQq~*-drKI~cS_b@>I$(ufoG)BUzsS2_N>3g4y5u>F^&t@uv%_uvRLb~1`& z0ZR}HGg?n0mUDoqh#_e++>#4t`8P65)SuJhSA~?EACI3K{?6|jzsn!T13uvXDjR}n zXuJUsy@bS}L)>V9Z%Qg#wBvw5CiDSH$BR`MC(yKqdQj5*QI!t zlMJ6vT20jG>aspH{{-AJB`r3iAaW>?9E{J(kAY6)c&P#kzE_Q{!8?YL{9m}8y3{1y ztY}@`s|7T)t{(R{+u1Mm+kIxoR=v>iVag}~kLD-%ByJ7QK(ggzWj zJ!giYr^ER0PiI0ElJPMS1~AhYyD7dU>M7GG+iVF)4%~%eBjFy+hI$^uk_an(tCj#2 zrcrNFX~x@;HpFO#6q`Qg+N#_ z0$ZGAL+tXLGU>LxGW83s-und1sm2Of;fK%JdCPfLY9$xldwxOnoUK$-&UvDKn_}0@ zWCCOQ)e}gDe&^&YsM|6>PPaRL-!}HIPkjM*R_`}?&n{e_@3-t9as2prcsQS*0~DE(H@1DCQbz*wPvq_>I>s1iJ5Tk-I;$Oc%VG82+A!q zatO*j{8?U5HeBXOVWqHOyBa2!mBW36EAX+RzreU!lC~Smk0)!6Egu8d4{cbd7w58T zzSyrZCdT)?d?yy4pk3r*PNs?s68N|INz=lM=IWPA3U3PFJJ`h~Z82c$T3R zs>B637LDyFr%Z(+AvsPc{+9=vt@JX70b~(r>Oo|&Y{*Fi0;YkYG$>RXZg}Bnay|fU zQ36?z_=*<2iWniGd2(X8AqQpJX3Ayvec@~yjvMK8b)Vl>;9KibO8nV^-!_Q=hI?9~4R=Nf@DM1}%}OxjrJh!~Sr;FfY)O z#ba^O_RJ(Z{MZdX&f{X)E+1}&r-VzG>W~nRV$0B{kTR@0$26dkn2ozkZi}vKrwKfp z!>7G;MhN5pF=${S2;}fufB+tZ?3%)Ib3op)5rNDdoNi6km#spq1`uJ3WiG#&%~X1i zk;|@^K=n;+Ziv{*U4-nSjl@JA5icigPC2@gpwStN@kct$r8&4&nH)C1qJ0-~pw8(B zMDmnYutq$&c4+c-X2iDFz2HaeL?$qn{+^^wA2J4u^|O1!9WEo~NFtC4V6`*sMk!am zWDHj~ocJ+z5|Ur&voMw(b`F#jH@1BGr`67Niqmj`4;g$ zxvKzGzXSk^NJ{c}WA8nUodPCq`^AH33FlhBPQTXt%x!e8AFSN$ya;DnFuGG3DKv)V zuUGNA;%rsP5C`|{^|dsJ4Pp|E1RV4?fn7E6?;)ok4ESfi4SpdJv9MD($JbI5jC?Ec z*25*YxwXKf^pen{Ws)vjy$9s z@k&?AN%hXR)O#>Q!AcK#7B3L`r8d$%15tcbZNQ|ma-Ar`Ur=N!5-%&@F zx5PY(`7L8xbGCmdHnPuc<@P|Dtap%cgdI_`qGnDKpSe6`@w-xPXwVa;W*1@Vu)M7P zVZ2w}K0SrNz8sJ&sIByF-4AJVR(n2+QW506V!0V+gE>|$%@JCZW&h;@%Jzl+(*7q; zZ-(303=GF~EZ>D7CTpFdd;!2wG(?@jHMGk%J=-g-&edR=&F*)r?mN@fmAxX!cq1+7tE1Qh!@uw7 z(#5OIPL<0lUa{4v44E*>t1|m@zd!VVo?W}h0NoOYl9kiV{bm2OfC&=z;lH|Cf@v7o zSv$+MVGlaHrKl7-!5G*)HF(wXn;$Q!lR&!zr-#Gt@6LCowgFHnC#1X3 zlU_QrH&=(^zxjP%^kVQJ9P(Wu7`*Iox~L%uezj@qjM^|XxioF zr!&lW##3om$#w2kFsZcvKs(lc);{&9Pg|@KM0v{6$ckJvS=OqgF38%km<2L42NYC4 zZ69qUjXb+>Tqe5$q3fccYEG)-^Y$l2_+>|{LbssujB;afynxSuI)Jy#A~Z~&%ayK{ z9QMjc-uRcZAzMviMY_>7Nra^xq%bINylFnhz(;~B_OKB3Afz%(wG4HIPjhc>*_2L+ zxQ}s(?@`;LK~yM8A+44KxK88=SB&T5@-jvo=-xV|QsLiul~v7-QOLNuHMO$q2N za+Eho2)JAS&bN&V>Oqbp7GfNtL|P$=CyB8`YA#&boa*x2)y1j>;*w1nMT0c#v_8Rg zOvF&EvUAzA{10z2SRq(5X&4dgkf~OV!y3=RGwghY{ofLX-Bc3yysq*Q7SJO|^ss>` zSk0;+{mr*(hmul(%w6+!QL!Kw>w)+@eO&0}I(^&=UvBjgRdDP|qh`>H)EAgQm&yjv z3-#LA|Ab4kKCYE*AJ_8SufFV8W%i%bf5OVuW=NUf$ByFUA;o5PkNOGz>K&IamI0D_E#=9C4LK z<~`S5B9TuY_!eFq9%mN%s!kTJlZKBs__$lPbt#DO{!(+A>tTowJuWuBkTMB{yhc0_U2Utmi;w!tc4)X{=%l>j86|^dClY(*S*ik) zH5eC0$TKr*crq@xM??#Kp(*Mu+3mo~HEdNq0R?w<@p?3G;3?%Cp57<){r)Th^wHr7 zD>njhUywztxW+aq-oqi2T%{>qmGtc*Iy*VdB}XV{=&9EtS{h?=M8}W5!8s|9+Imt4 ztN5b9mC{seykr}wBFv{5LAv9uN4I&6pRJ}5`{SZynwl49Vs}u)XT(8-&Q-Ab@`x&{ zK(bs#J1Oz6I<-$E1PjDG)`>*@Y=S*0Pp!{e(%&|m0~LoEM=^OvCQP$Xyjva_)M+Z{ zHoy}i@pZcSJ;}thTx3?8kPSkBDre;~=VF35%(znuja*9ENKZ11Evh@+AdBDRk9bYH zatpEkc}Qufi|J7xF(OcQNeI~o={Qk^T+oR&Tq!SfRc0;uqC~PkCRrWN4a!thAQ3XF zJGiKa5cyY%C!B5?b4-C8af_pFXXMdIK+sF9L@wy z?m`4eh6+Tov~Jy7tG5tX$kdNBb0L#)TS`OW=Pow7my|KGwlR*%2&zc7R+?u1dE_zo zJH83@&uk@%DW%vHG7}+8a2roS37ZQ%AHDga&nu9JST1b5+vaMi_Bk}sT3QETA6B@j z;csX9KD^63r1uQZEyhI2A%b}WyBkNIfEXl}JnppB4d9|MXxTQku*;FdgSlL{sQi;g zpIU)Kkwz$OtQ!o~>%}evzT`w*yY8|L}sj2COWIfk(Jw1;PvSC*NH3A?UkG7 zRWe5Mov-J#&i_Z*T)?MjS%8TpKh%Ki<@evglIF4@`9-)iCwx;I2lVc(?c%z+j|Y5v zllJ#!@~J#olj0*Ckgj}lhHlg@Pf{-a9B2sUldTbYJJA*A_$sH{R?DuZ%4Q>(GzQ>y zKNQ@)zQmXY59nid?jM$iwqZBhwPER6x{wuQnCwF2I$q9u z8!FWz&8c^#P&?lMHcClv$H8+#N9JxSUpR4(O9r*)<;fhskrR>?-JB4SMJ8i;YsT*J zP4n(C{^n;;hMx|!?YHRf=$q>424(-*J>FTz%W_k-e}MmgI{wx-aQ2_7SccZO47AP9 zjBU98EUka7+kUN+eswax&f8oCzm8*YAnZS(&mD(Jf&ZKLKmU#YZ}aP7-UEDA`cKG# z=(OfAkd#H!o%2OIJEt}qw-Fr^)Kd+Qgil{zMb`u-I` zkgqjnQC~Q2c{PdOMTjWgH>JAu)-Ztixs&$zn@7jGlHGDj`yd`@A;-c?y z?I;(g)VZrW5g}rGJsLt)QuPBa8Eok+DK55Q_0ooms3)U<&M?X>CVM&8V?8fdq1~xM zyrt>$PS$iO4!ET$eR#Ea#nji_a0R{&V%o$`E5nW~n?3hGta_aSe3%1xo&GSb^dw04Z3lOrGPo6nTo6X#f=s?Ipm_AVUJ3)Z&;Ft~og3BO+x4cqh0m z{n?tQMf%7tmO@Y6n+;LH@b}k#y{%`YP(NXrcGwS%UN;{(K5}*6!qly-ZDuL5g*jKq ziwr}D-KSNt_4+T$P7d$3!vv4HL((5?Rk`fY)>P57OZK`zD>`4*M%y@1c?!&uKMT%l zZu0h?7_}%v^-qVy$gV(wS!&k*K$) z+No_s#Aqv-N4Xi12;@jRgmf*!VD7JH!t>0bs&4ge({R-R%CFTHB8iCGeK^tqCzd%{ zMv+=w4asuLd&EB%)<$EFI&-hg6>iiYe}$=e-c-et$y2e1a~SX-%oxkwen3~ zBzYgFItxtfFU5&6-LyK@n9qUE)6lh=iaqM5ZHExeLcEU@rmHjK1Y!V(pi4-4-O}4{ z5m}d-zlz{P!)6;RtfkaRj}zAjq(srf5GY2;W*pm6nFNik)VvHBW|Wkqk97)wFVTx5 zvJaMg8MN#P3nB8orU(b1tbrC*^8(mwtw|=SkOJEf7o}$9)nKkzx_-J)bsC&W3?NM- zqo#uc=CP$$<($Sr528^-gtH8q$xfq=5>QI2kNDUY_|iXt0}we$9I3;|m++jb4hIE- zPiD16k-WG0^c&qT*S4cGLL6@zdBC2o#iIsc0TimmeNUWaQum#AYHVa|FdK@%@)oYHu2of601NH*#+fML?q$%S>kSTT&&}nNtOZpZ zMQ@j`faBJ*xWGYh^Pvek=xNE!+pDA(;&}7ID^T>d9%fJOQCo}4TUzC&EOa$lTX0b> zO%W?b^nQVxqR-2Q*QheeG`LtCBh&}RwVs3C``L6ZS^5`{yz2eO#nHFj6hd-c@IXTR z#rAYbFzf=9)G5)0M#IGNd_3lf0YJ@V{q+5qp_|CtmGbCxEPAe(72C~gXwhk zaQ)2i@e(`Hw3sD&=9Y51ID6TQ7fB$YG#?S4&`^pbR}E!rA6u{rTd#qpyUdqG^((Z# z7gfORgOsT$N(jauL-Zw_!3b`}W z&%s1#Azk3}bjCpAqsa?w5b9Pm)zVQCI9!BN+B=`^f4chWb8W5LfAYzor9_X}MO~_G zaPj=Gv?r7wLIUxGR)g0P+LLB&YVolv()O3w6MAH4N8WHuXr1ZlJ>PabsQFJgBG3Mn z?DqfksNn4EV&UOWXMleZ^YEx7eng{3kOiSG>}-_nGjB>czY8Bbi(Idl39DJBnj|KF z&U+*&I)gHY)D@JZBV&{(h;uV+zSxE*zrdARTH0D(>HXA zP03h%!&nUV9Pl0eL!uB>3Sun28dapKx`3ID5NzUrKq^dv%E|sQXE^Wgp;-}$xgU2@ zGYiQjZ^Q#)%|FZ%0D;VaNq0s$vdpl+iQ0-B?hxY}n@BHeOStrk140DJnLzm0JZV}xQ#g!WnMDjMAWLQQI9NvW**Ausk4{oZmTob!nd zM;n`3jFbFt4E{pqJsedB6S-(|g%i;Ji~I3mQ5zCT_zVxzryi?67-T7&jkb}o}k3hkDjtyeNnCuo)P53vURnzd| zMQ8+@e!sq2vS*A%;<0jEHKupErdTPXO{32$*WS@Pi%=py$|gqDtNk2rR&m4;BY zFlspfCsN1p&w0x8D*)7>Ki4|gvbd>K;?!vuRtR(x))c__)B6anMmz$=T@=OuaZ@@S z!9^W4s;raMR^O*iRh%KB$F;}b_QtN!rPt?fMm#gf#vZU{BB*34TP5shOP;dka#XSJ z^gUP?yHmm^yA6~JZaoKKNw(Mv`vIlXa*NBrA!-F`wi|mds0_=t!9>iv5eaB(D8nJL z)(HwU=Uu6$k!9UVyOydVS7kCqit#$sI?f*;9Qj23hBI~g$eE2|5Im&z+WcueqO>KG zR02BW~+$vbHS?|@0g*wJ9i95HZSbOrsXxxn5Gg$Pq;rlk`kgVC3;N>D}vj!>>X zgyiryX$5o~nLeBdC6p5!OFW6dgCPNN8q^Lc_pnK;672Jtkas6s+Ql5`rxUyodb3jg zlSDk%_9njzK2QBIC(Sd zLqQNtb*I+OFa2{Y@!J{dI#X>@I`=dj+Q<%L*=iwjZSVLfLpTA^{yHq%3L%=U$Ug%-yt_rhUnau>Dv4R1AW&pm&WCC(gs2#Eq zijsj^O+G6^I5pWG-eQFYiLPxSoFB|=unAQsjz6iCxP)$Nwr(RcNFZz`X2hY%7>`eN zyk5pZtJ7Xe9}x_G<_7lX>Om@=u6+yhGm?m} z3hKlvox}-iOM8snv1}(wsi?5Lnl~@mx5m*SXv!dIwVvZ21$NPIp?=ByqY*a5t7^5PaU=PsJN<&}rU*VJha{l|p zGEq0Th|PmJ%{8r%uFe-{2eu<|}J;Q}ZG(uUC&2ZQ9ui6H--xq;iX9Z52_baYgV~c@ZxNmI-|A9uIBN zTkZm?wSyNyqBGl{Mi6Und7HLL=c_#^#^5`ag2P+L*y_CknhR&yfJ}$Wnn~1DR1n!3 z1cRJS2BAxwTx4}&hhSu0HP5a>2?$t{rp{|?Wo8ueY$I?B;Z@PlJp~f{PIh&zG={Ya zSqOqH1!0XhDd6`bxH`z?#&lcH1Am`m8XWAjEYSetQgsC1oWo#*)N&{ zpiz!RZ?qluCVs$5V{7}`mQyE?gHo6hvUR?9&wKdR?5R5)fp3Ict7GlnF-JOv-&$s{6j2mTdd8^V#!232EN`3OZ7v73Tux))s{A(@ zza-2#`;@C-{qkSPO%Z?ZN{eeqo$O0T9P+fH^wJj`CQ_E|oDjatjF`1)K4omzLCkK8jQ{#3cEoLk z8ljk^Xdx!(kQ^1Xv$Gf=vSxBLwVfRa!}*?$&&0+Y0OH)Aqc0CQyi(6Qv4^jBojh|~ z7VB`p3Kl-11E07*%zNLtwdS);&oR*hE?@(^N1RAK;9erV4xibVSPPr_e~eu5N!yB2xqBn| zh698UKc?2e+Ew`$S#~4QGjE5X7hleCz*n18kn);PBxHarw9MKVM8@LZO_6z&e|jkl z^oAX!HUVjj>Pmf5rvqrm#pp`n2%qrl zP8Zgq&hrnaKr{qW?yz7WU?eG}T>kEBp2$Fdx)x5gSw+8Uj4r}OfiXq3;H^qyTN>Ag z@Fn2gCgoXd45DD=^M1r2lBaP@Bf_cIJ^Wks zxpT&+6Fsj`vtnmJFt7KQA6Z+y>F!TQRU>gLW~odjI&nIpG}A~6ST^D;iQjX=3HO-7>QZGG5X3PD z$gFmy&V0+UXxAXY7z=b#*aEWW6QV;*Wz#xWI;&Kg{HKs$eS9=hOsp*T0Uc}JkrcnC z6Lqf;+El{Q-FCF$6?@p~6Kp>v6G4*?FH*}R^x{IRar%EE+ZytvJO=9iEz8YMplyS+ zL?bJp=y@hz(c@!AI8*?!QK(8dZCS#B?QSm>0v?;33TXkrg!?oK+O|`RT!$UUogayM4q`L86Ki=OL-*Pi0ejjg9 zIi~#h-mpJ!C0}1RzCIrp>^-_)p6?fLfIsiM7d`l&bpdS4lJLHmJB-O$6lo?jt|CTm zj}Bg!vP781fd}ZU|AsgUivlABH=qx*0E;>eFhRgdXqqDB{uam|iY>xeN9?kKTiL?O(+hi9kGqs7%>@Fvl#tpCnf4u{;yjT3N z`HQVcM+CRqM&M0&W%ISkaf30O!W@CNsH;|Li9mHqad)p`R#Y0B&lO0ex(jSK9y__(B`ILUyr zT$G;#*!@@4!=g9}75Ny#dHnTV-x2$QE0qsE0M(MqeM=GhkdRB9JA`j7(nm} zZ+A@G3O0}xacxSCoVKS4dE4gj!b3M3Y#F!`oN@cp-p#npS7Hjp^Wlq3vFmR8AEHeu zMJ;qWZ6P6%T!O^M>-92kpk>E9vnR3Lv<(RAl(%`8iahbtj*oDRkyjl;f@R(>TWn;1v~1PhP?m zQr-|hjmYy+sO(urks|GnpmOfXlqJjw2$@u+I`Q(>LV7``nTz{&okK{gtV5~HU^3~n zD$d=Sxa@uE%zd=|fN)dDPjKgp+JetG_Ta4`e5Cg1$MnK$e**4i9(nTyF!J`l<*o3d zt2na$IH!D)_o+iQNz!tN8;WGUXeUK8Z^+dJ^c$3O7U92DxIQ!R0}8`k{1c$D(E5 zfl5=Qs0!swuVh9WXen6e8oZvzS0@mb?pkm9;0+fQ!j5Ki%DZAx@M4<9$Ei%%y(^yXp`@(h^BQKUE5X-)YMD#Z1fFb4RmfVJbK zgO1zG-jg!c`jOSF*r`O0V)U%BpcOZb9VA@!wS+H*E9{|fy4GbNISYaz1PfbKcZ27> zEoq^}zU5N@#f){Ui6uPj8Six@ji~7Hk1WwC&N%%z@Xz9FVQ_lD*SB_dB-sZvT1W8? zatf(PV|*mtS}@Mqo)Xn*LYbxkQXdLij|f-=2E*0@LV7ZvHhbaKff{MxiZ=xB^u+Q< zu0gH>)1FIAk|shg@SfKb0rm%KV!fOF=l6b$lKyz~nLzupiIeh=gklNAyw*4}b>ZYV z2{jsHRav06g<1b8PskT=WT5seeUGU!4hXdpow&TD z@xzd>kPh(5$Ff~krSX2d`0Ki#lj$`E^5G@UvmttL7t)2>p(QOu!E)@>%5*p{-1ru+ z2nbE`d{l^!7R{y^>U>+LNo@^>D(x2*mu33XsO!x*;|Xnob!~`aSvI^D0)Z>QU=GPa z8-UYQO(&h+R{)Qddb!Wl=GVjdRr5BZ5ov9h_REcq3Oe^>c98>vBEfvSqwJIAmevHY zb9LkLon)Hc3{rcbnRJ~HOE#*(mcG+vm= zV`C{oRH_g+Dcde@`p`7VdP1>uY++r`vMYt6nAHby=F+KQsu+KS48C?B46Qz?Pru|< z@Qj0pol9~b$CAD3JzS17&tZnDd`Ci&#n0-+a!$cs9GZrK+bz*iod8`SO3k-a|IP=H zsI)|Btq1|iImohdH!y3GeY`Co}G&PoG0Hi`dj^`Vw3- z1hjM#NiRvlt5PLIZbEpRTu{456>D3I^E;WM%P)_c*W%KK>*Jgk?P&tGu84>?fyT{t zMvKXvsVYa=CTVVlS_t1yHU*dJITNBGsh#wV|F{T$_{#cvI{v8=NMX>_(y0q+=(C-9 zAh9U#+Kx#9z)m=l*dI$v!F`rQ^s^IsZrwGFJYiI!iDxmjguk`4yQnd7_~3H&!91LE zo3&#fC$v_^{|5YavsB}TRfSG_XJk$!XGLD7K3BTG)R@AL%Po_fg1=d{iD|KroJPAT zR*y=rE4=)bBovCva3EsnPV^+TF$i#qHi$e9Wb+{;l<40iON9$RTa%D5JmJ zooV`3cbZysPJ2jgTlfs3gP213HZ)so-7P~=;;O2c{iGn>jfpm|w|H_fH*M>hOsS9@ z11FZ~hY2UGHun10h!1j4pm)dG+OqeJAey@scqZM)im`>yR(LA1wL+v$`#6pUjfSab ziBNO}xdLso((#? z4v@t|UEF)S&Ln8L6ctVx1vHkvNlB^sZl>ME@9`6fGgMbmlVR07iaa9=IrR!ZicNMH z8s#qJi0Ai(C{rHhXRc_}|CB?K7H`G9#B(Asxkw!*w6e`b8l50vNKb=Ik}Ms?3k-{% zcaWfUk&vOnmm|sqo6Ni9T?QXi2j8~Lxe%uJ{bc9ok$b?>|LI-IEfEm?+h&X%S+iq2)@FpKpJx)n?V4iCC{sriu86tu6^f z7Mzy)3&zG#LsZ6byR-j3TsIbz6UmBZwoD?ETCXT#684`xtQoQ&X@l{9HLF14@;k{C-46?lcaY3V@M=!dBBN zo=Q{YF|?x&IWQYYUgro0cH9%{-^CT#jp{^I=1XDAl3zGN1#Y|CRpV66`xx;b(UZ1H z26vDVK(JcFLMOM)6&>JQvRs8cWUAIjocu^_8V34wo+yhKM>Vr$m#nM;69~;)ygB z>WuHAGe>gX z^w`2^pr&iQD_}SlLg^Ymim5(BG{R3YCg{l?vPPDtEUgt`=Ip6rOfqHRC5u}D2@N>k zeJ=jl^ZT^6Nf=#*G4j5i4PFgxt%5v6z1{j>ail0EpgO&v)1%xLnUlNe(SWqT5U`}U zB{$V{-wxm3ms?BXf?`mL9=`+cojjjFdXq2N^HqMh-SS{{EIF+TXdJ(-YoV<5D?sQ> zkp+Nj#9qeZv9eDHW70}u@2>it8}^bJ8O!&4nLlV++9vDcHt+CtU~TNIqoQHu_db@J zvJyTyQzv;5U+9TDl*&V>ICY8IxtkAt;ls2c3Nq~fUEcw;&Vi% z5t?q1@egd=`AihourET1<-RzslggXQ$i&hzL`R25O_DW?(_o3?2OVRmqXLPbb%B+Z zq~$9J%7%oSXo6)t%NCy)zoG;b_NrF8&&V>m_1x3@adDqPb)M%omX;Il2VjRQr*YDA zn*nZCsim5^iFpT`eujiF&-?QNvcl)Wl=JwIOB)T_4!+Rd7xQG6f=~8#+ppMMs^i>Z z7#RVxcRdFyI>SH$L4c(E^)Yo|-M%OhDa?s)@!>@r?Q9@>#TmBWgy#vi>&82LK{>FL{GM-;gk&aY)gQUo;bklZ*cW zQA4hFi}28R?gND%OX1D~sD>ZPK>?dnIVAZ5>VPMzk#iDzWDyYBpje9udcq8!KSNZ5 z&p~*yw$VqSCY=Q>bm$PJ1}|hGudiwj%kixxpL>FS05;)j`pDa zD`PvV1~rLe;1DS-gNB`sc2g%jd)MqG%~{6&w%r=x8T~7!Qo{cbr00j^;TiPv?(#$K`tCF%2nW}_ZQG~t zqCAR)Zc4R;BKQNfKko%IpR}vtQ~q+)kNliqiCTFIDenV;Zy}LXsnz#KpBO{0k@^54 z_IN5B`;v7R>P&yhm_EW`ZSWI+|$skk%#J0j*L2G&fhmTy8X7A<` z?;h_DbyZ(}0Zz=4d5?H3#g5M7ynPG-3kC`xh!7a$79V%l26RBv_ zWwE!s%t;nyWQUK5dy1$xv;Yj}50`azh>4BXjsyVGfQa4AtsC8@m5Q`f%sm z4YgkI=}Wrm>b~r($mUjsj22}#q<;lZG!m&5S@ESsmxSH2#OWt4KNR$9BRyRwQWZ7S z8)w@{*8$bl=j8_XPm_e-Fml4OM6Od8KOd7zLyaXym$i+8@#(xtouyIm>3fV@Wm1~Y z@Vx*Ig_T4mSU6--sT0!tZ5YLi0}#F-uUr8dLziG9C`$xMRT&s*`5o80P_iG7jemd{ ziuxW8EILX^!WS%BN6Div<57fZ)-EBKg(!|iHi=%zJc-d}!39xFXHsYo3QDXl7KFB( zmMEj%(C#CKWm20yH+)X-xpCp0_DVm8(?x1Ts1+t|+2M-O#H=u^OGq(= zek$Bvb@xl3sXFY-;WN;~*Da`q&jWj^u;s zz{tYXLaKC^bYmrT=SsI$+mmQ~YjtTTje^`>u*hzs8hzzLR>;};jQm69P<5Tot?KP+ zceMKv%IBG#?wO?t7Hr=#8bfgup$(%w!V3qYUNFaC@Ezyc$ds(y605nl?zsEiV2d_B z-A}O$udW++#}{)8U2*qCc^_@%ns8%19pphrB`$D7t%dii&#=%WGALMwJU9S{ICm%l z!s&_anrhtuysY#ZQ6UA7Xu^MCG08J2TtFmnp@?cW1)r@ju0~OrUka6ir38{*)JwvZ z^nTvd0|KY%b)-J<)3?w1!d8^S4WVuqjyIiE`KnWP%T6Wc^ zG)_1Z6s&~I*+MGj%3+u0l3j|_nnhB?!^Of&{I2L4UEFOqV0~fyn)&TU`yfwyi@oh< z2r2B3ZEknFLO2s;momB;sDJgi@Fx}{BQ(`r9iQVQ_ORZ)zbf;XASR8x@#d&+oiWd< z&o?aV=Ig%FgZG$Ll>1X~FTnCWj%LTUZJRr`ZQHhO+qP{x`NXzu?%26?&iQ@c@BgK1 z&Gf9Am#Lbr?zQmB43-4!$&rwYw<@We2x0O?+>ufSpkV=A6ioD#@&j_&Ftlrsimr-k z8f3!W?-XmouRajD>aksB@;hZ}1UJcT%uy6<@ySFbG%;LJ6whfjnO{umB zPSN`gaB@gviI&9_!=ni!d zCo&EfouYLzGwm5;%=1Jaqu5}TE=G{dSrj!9+ zAS9>{8ATZ0MSc(C(rusVssq0{?CdOQ*V;w6&{XL|E|;E5>IPI zO`+nWEfPlo%mOMD^``AjG>@4ATh_x!9==IH-8o&$%1m)Y<7!%ziSXmMlUhyD`R-%U z&7(l{`-3TkK{sh>>EuW9f<66gJP5N?XnB#&_9Q5wz(2C70e3;=Pl*IooE)^SDzhT8 z8tiP+=^TL-Pf2_#hi6_=W{DY>>~*&APM6xH9%B6QhQ8{oL{e#sjdxgs54wcXBeZBc z`uXT3&Xqp1n!11pQQ8wj=$DNh*IWmA>P_*}n-q8Ng>ihl^>W9-m)s4%K|yrCigwTqV?R!rK+`PKAE#w@ZO9X@BgNbUS78Yv=?S zI4t!sE`VOo#Sxc2J4M{D;BS28d`m|3*a3zwqk?Utg`@|OI(gkq6nv9uZv*rfcA&D@leB{Yv%*`W5R?F=|^z}@38NOQD^@7@>c5rGO??8 zBPJ(j*X)?>=xCnGtl0UK@ z3|*{zaGDz4ujWvrO}byWhnXMiDJl6Q|L@uxTF?z&!iefl)4g#w%|SL%xq4*)cYC(uu{@ws*AV4$r=!-{JJ>f*n*(G2&5V zc2P%WxBSNOI2^K=D4yX~4fvL{&IfDD{2^^Km^xAqW~$Ha)*~g$ig{W$OMiErW~i7A zE{54Nn#aq6U+J=Ay7+~st9U3bn)WydU%e=6x{?g&wh}_bzey2w3OOLTp2T6Z3~D{8 znq=lLX-lgc*Ihx%q@O~_P)r(~`(|+H{i=2nw09$d7)r&uIZ2!>EinZTPeLf<=RA!b zFh~gMtq?-!FBX-J_NRCM3~0oXTqvjGkBHDV93xDjms5%1L@GUPfq z5WgD}(K~XV7a#E7*O>XgYHAs@he|~=0X2XHMc9)^QlJn6@F8VvelHqh2(0}~gG14> zy4a#HI!_U$va}eJWf7XBMYhNiCC{*vZeuHp$MgD9;z|mj7a=Xnqo!S9ZKcqAeqHBq zjJe;BhdH{W`{XA&8l!`OGXl_gWdwB|ckeMRiI`KP%%?>duUC4SbhZLc+44N)2ZZ{K z4sWtx_QB)pZ4%3Lvp-G(8$$?N&!(ndBOyE#zhbu&{egK)3q8gD1H$^X3s!sPXCOdt z#bh8aG%z3}6FJ<0nL9XTHS@>&ZOK2vZbk9FL%y){=FwIS!|-#M;J<2CkF&bapv^eO05=L`j5mn?TOn5;4 z6&jOwKXjD%*e;I3j%E#6K}nd~D~gD^N3)*Om+Ih@w<4fj{;kYwr%_%UmvCc!rQ))5 z)dUUCUFpom1p)z#3~hgWPrQ%Jg2#%|$9MN~7pDANU%q{9zz-~y``d-Wu+ebkS0&oz zpUA?{p7T0-Pw9?YO!G?-_9JA|nBk%?zQ+F<*#u@$I0LI_4_A$m^rs;kk0ORke zIqc(}7l6_KHOBC#*H+9p!@vA5W7s zbPxh(z~HYrY);l`rB}z_rk;|3`L;={%ULdM8ZVGf$_^R4DP}@Z@c5M!j3Nwf>Qe{o z%68>Zy4Ni!U$Z@2vZfnlnztLEv4k*-LWW%PtG``{{1Q=N`doZC2=0T|;kkYrteZj~C7;B~3!IL{!bL8cB7+bhWci!)8&~Nz5_9>A_rFz{H!I z)4PrBNw9ghM_OwzXMsqjG8@!|pv5v6t!(43bO7luBh>cp4>2dwza<2Yh>r^Z}ko zhWE0j-kCAxY?zYAMPsyegH@)u^LXq$s>?;Q*T8lHWJP7iluv07Blw>2l5I=7T{FbJ za6;cY4ZPd+!BWHSoyy_j?XOxV^Xsm_?~zpYv$HJ-vA)ySp7Jesh9I~`e{c39LPnj{ z?3{!gb#xPyo@R~ZxAvc_?m~*oM zcPG3mbLl19bQ&DSU6wSA3dg`Zpp&(mi%a6}TBXDCtuF_ex#=11|Yc$O^upL&vspl z6{Jm-&M^E3%D2bk>aeI{C;MjzK=suJQ~V2?*kNOa{x~`m2fq;{RFz^kx4nu#lcWj> z54Gf3m^Hydb0x0A3v=a)OJ+v)cv8hvYIl!lJ5XxKQP-Pna}>}@RzPedF-10fi9+m~;!@nc4aa6H|9f}s99H#;N zsnGbuM7WOztn1JukTzWRJ<`3A-eIU7Rz2^|j~a(%`DCZQ zHei|J4CWE^4$Z1u*%(T4;-M+1GI-y}+3p5kSbCq!*;@>OLh*5gpRK$v9@|`oUEv_R zGFy@Xkd^a25eD!#dOu=%V>?tjkJ@_d~g)Z?wP0Q9`*7tfGuHLI_@A(TfQl5o+)ZX6Z;|UaU_;&s^e8Nx9 zkDJ@W>%DecpAz=Y-W>WD8BNU;CUpplDcK9Wj7CBTv9ZTG6m}8ITP=pY*&@qSoI3SV zX97%KOV&Y{(bZUkV9D|LM`sZy25h(O1pFyPSfmQCOxL_Z1kWI67{I)VbOAM2RFB7+ zh$LIkA^Yl>0I@F2F@{h?Rsq2*Ng69y=d8|d#s|0M48myXCg&Eq>xQ`$?qCvy7UE$H zExB3k_Y>N3w>bK5nmFz0b3xT?rw5>Sh#xBYM8&?|aL z=c`Up!32RZDAYpAzQx&cW3ME{SR6~VAo0?jt(%c^J$6WrvlBdRFLeYjn3NkpZ zo*;?D>pC34$Y}iibE^55)5Pp>-pO6AGCsxCT+w~LP^W`f@Tk$}FpDg7vZ;AHIa)`B zW+Y8Hp(R;97b9?Jg+bUk!Dyt97#2cUs{j-y5-q?B0~eO+EQtJLDY4&gGHm5+cT(p` zUgP3#t&spS3sG$Hw;*~F%v)*!4(T{&&3!EVstZ~JNgZ#`T(qX*^d#F>OU_?66)rLK zrk$Ho%viyf=?YE5I7O*vxafZfyhX9mnCOfGpJaNP8gB`x^Bqja(7?J~r2Q1)8M6)S z*!>t2A3{=(T@`+*Iasg>4^o4?$tzjRYTWmevnjU@bmiPy;muQW0zIm2&SY|7}~-(5ztLuOwpMUe0YFs_8tPpt6bEV%UkI7S?K3gLBbKz z2FAbEcwl1$I@>z#p^~|D12Pd1FS>l@kzhrt355m>Y!)@ARJzPUvVjp|$mA|{+ny~Q ztOohJ)rV#yK|-bVd()wiP%A_<{0T1Uf=44!3M7t9bZVc@J2f!B#p5q>l} zcGK{;nitk}M~9Oh%#%C;j-Q;D6nBS>T3*dIogEa${yB5`URw1m(L|YMyl&lk#x*RFaw4iQ`P7BV|hV-@E*HFZIMAk#hW@m z#T{LXa3iAuaL_Qbl^RNsMo{JVt=?_{eH+BzD~e}GZLVsj8>@!q_SwPcyB=?UH1IN| zMu5k0oP(?npFYJs`LWvp$ZZ{x&K<}uUDeAYjJ;PVgnaV_@h*Nj1tmW>e`9foe1ZE{ z1c`SbXH)y6L*sJctzdFVyJ(->`#q(mi`y3ZZZ(^1m)W@jE}#DiVWZOmJ?0kk%9F-M zDYjhlR821dq$kkK$LY<9qAU3R41HqK;9UrSY7IL^-KH<;q0D*xdzIcxc_#g1zVLh5 z?jgV-5!JD=AQ!qg*R*VnTtej3-JLaJ_8Iu$YL{N? zk13;oxuZ*Ej~Yo_7M}WWtNFYQCu-9F9|NczCX6`)OK(0No>D_ByN zbVaUqsHETYEwPQ`f$7G+Tjh#dA4*vYKX(nOto+_~ZM`$5y-Cey=imjIeZ?(Oi0hex z66PA)Xq3Kl3Jaa_Un!UI6)j)e)~ix>N+@cSn~kX5tp6W4O09->r<$|TGh6K-NGO|{ zh!x?Or#9I1k&{K-nlx!NS-R1+kwB-}G+bkBkOv);KsIxjAR7Rjo9b|6Dq;-)C*RKj~?yfFEbqxpjZL|6cVtx9&g7 z`#=7_FZh;X{6tkI`S5O@4pbo-0!c5mZ26UM}iXoBhLu652CV8mr)`zp6qtl>K zaUF~=VTQVDbf~PVGQvc#f^vKPc7T9BGRZDuGDF!r(Pn3-zd2VAthgP;B32l8&qm~v+qd$@)JQDoO9a8ds_iJwD z2U!gTZ;7w#KYs*dN#eJXr5xO-*gMOlaoSyAbW@<2a}vTR{7{k^8d?z`krsUfsWRoj3G>>}93O`#Gd(EZ z`+Nj|>lenqZI;C8Gx{&LD(jU)YX-M+T?ucOd?sXf3J{xO6NG!j$(nfva}jHzT*h@9 zW3h&Q|Nfz zw?8;_49S4WN;TC&o@l(nr)*b13}L-R3GF@2Yx5oV*?;t#>mNuEUSNN4f-5yT`9=0l(Vd*Py7Q=!rXH4wrFKt;`ik8)22?a zT6>nU6&^%<)4IDXUNk=D@v}S-3dt{uvpMpo)?*<|z@&TlG@hNpz(M`5_8w0^$OW1O6u?U6jYE?Unih}kt!Gv;@am+Mwi;a=JCF~5EUajmYNTCV~ zO~lf3bG^cVO`We9W7j7;?zyv`SYFYir-;F6HlUDPxVfDWUDx#lqV-q z(HxgU$Hrijom{a$x_VQer_&aKor!2GA`(f=&~|baGPiXaWbQuuA`$QsEkaEP^j}bG zw?W#|?zs4}CseWB#MU5?Df>C8eiJhrQV&f`d%|Z~9a+6@QMfuDwzAR^<*6w7Go~0K z4%Rcz)zs)KO!BWu6o%1yXL6#>Rn;;_LZ4mA-@T26^PFiijH0=8gWk%zaUd&7PeUzn z%55cO*}?nfc*%dVH`1++`2ISJy;Lfl7~&yDlNHHTNub{k&9^Oe79isMZgGII6KQEK!z$Aj1OH>35MCd|yB zR3S(F?HIX9^t6LSY1ffV)M5?ZM)poEaHT^*S~`yZ&W+XVRWV6BsuQH<+!&V+em!(ig2oxxoHyM5#FV{XXFFNPL{vs|*V z+T;cXBe)&EJx0JC{usKqDnZX8Em=hzOtdNlxTh6jwlJTKisNZ zvWe*xl*;Gc@M~vv!ftL8zjg)v5k&kIP-mh3rgR7$Q(H|L;D|sA_(qXks@dHEt%6}) z3>Z>B>D*%mOT<4uxvu7)%QJRGa2s2`!-3mHg=l>|@)uWr0oS?Y76m;kWkC8<{3>p< zj>c}2vo-q<7}$v)+{nP@Bm#?jIA$h!i^?j11W`(sACzd|L+2#`8Xnd;W9LoOOOn`> zy?g<#(6L6twj;h8>)^ZLEMdR7{2OB{+rC+j-02)$|PPE6%}d!ALA zr;>9n8Urugg>zl4A{Zx#Ay4tne^`CaryoM6ltX? zGm?ByCcw48{b~!`EDX^rLtej6I5%D|8U+`j4#+?Zey6FPEIy|mi9yn?``=h3JICRw ztNpg{LGq0Ve`y}l&w|f&Cr81G9i30zUWq<&fg4jB4DMhzQ!7m6CbC3kmn|wa6zLW?v3ouMuWE_1vN!;RBkMD)YO%OYb1K9EW&E6EQjg(T2k!#A*&<11+NR*l(1|Le4YiZdsm-wOB@R_&9%M%(2JiFu8D>J3S#Kg^(P+jYDb(F1 z4Z8u&DTWuV!>&@(s-ZPggcYgOf)Mb>I-ODqdxs(S4~-nU5B#o7>&nO?J%l?rrQ~=3 z9dP^^b_};z!f*3vHv$&7awvOthWpllSB~r`46Ct7W4ca;5U5M~K-Y_x53o|Fok3pn zoHD(I*faiC)EKI9TE-$Y8@_iEtYg?8ua`8j&jF}nUkKAtj!U1vLY6;rAd*{Seq$Rt z-K_4LGtyLQGR^e}+<28Mhfhrm9Wob993dczzo(p)JAyU#pro?cU~UPzu45^2MCTiy zFKD)kPel|@nw|aAy{7eC-F!fmJ=fV~H)9a0kxjdgNP;Yi!jA@{12!f7Ou&UNe_%u}q{Ms4QyObCaX_{D{#L}i{Vy8Q9^ z6n7B3T+i5N)mx{#D-G!r;TZPL6>*(?jw9wWou39{z6B62)Mg0XTSQ%#4dS<6fxqES z4^uEsiWA`UWTT~FlLz)%eumY;^0d#j`WW1|Nq>P7vlQQn3q}e&TE5+y9bgcP)8^JT zoMI+|Q#o11twuJZdK^hx{~MLKQ}8PWWt~ww9VsPOcs33D39)`SQY&Od)?% z6I*TMMXNrCtgNO^r=SgQ4!!-)?qHErV?M&6J&h_~c{txVr79TU)!KkU1sSiIB{>|@ z%;&LCPKSjX)7ew&sB%ZyTqrSI2z4C9Qu+%-(dvwZY)g8_gU*Vh-0fJ437QOZfE+~9 zm?37&mZDv@%Hl+-qr{6%_U&4Wg&Ag4F^BP1WquqH!6V7uNAao059;Nt-~$IgtyB5& z;WXNTNX(eRIgDHWnx=bh2Cw_);FLncb-Fyj{(53(Dkah^&-tVa>+Ln*1h1FdatN?m zGu;d!@b?8FG`oGhshOuOMvBX6#9LI>yW*w_Q`-?bDi8>C+@!suT({Yi>M6=x zK>&4xrStqRkdF_A3obs>lZ?zbx8?P3&aShqg4*9inf_$(p9hP#{Y5u0JPN>H=RzeH zJV1;2sFMld?a0 z_h|tzY@Ik0<%$GBY(y}UiurGQY05xtKLkmNbppT+&lf{`Ij_u^WsXI#QlBZD25ALQ zL&(SaPK~3b6vyy3J!!ibo@R4)>mM5>rCmB6viZ4@=Avw!mnSqCBqCIN!Fzc#;4Clr zmJo3KYV#DOnm8{hLr2CY00nl|I$ozBa@`6HgrAWy>DW*&4F={9^-x;0&&{V$u+Ff< zny$m?XP}O}j{J5*<)4z{o`j~WuTgZCZPGyLjyGU9EYBpHQ5`iEtn1i|>u6cN=B^4< z2WI9%XS9)8wwWe%_hx#{k_;l)^##y}J)Kjs{l?jnd7oJnyVczn@b9+Z*&z+7uR62) z69!x-RqR)|zWEu0?v1Rn37hw+*53A5_pPp>|5uNW}GydE61ovjcV=p(A3r2j(-l|t(T?1PT#AZ9o zN^Dy`@cT&{=vQK?c~9%w(OhCLWQ zrxsRErc{$+pmXu(;CFHM_jYkn$c$tjyN1qYPr`1@S^fCL$=Wg$iheNyl`l3BpmFFz z;sP(iT)(jm&eElRBmCQaCQ86K;bLGxM`NpK`(t0CjgWTe((5~QR9lXZ+{A9kF%!N> zr?j+PQ*zkf(iVGJYZelWG(&7-a+MRgr{80Fg}tB^^Kqh|6{UEjT;`+6wGZY~5jCY; zQQ%hl{_s+L1=X3SJVVm0@`+&i=l;RV2&~EaL%MEV(dQfUn^Xz5X~Y7hy6z7ZHeb%u ztyN7y;Gp&XuI?HUc~}(=p;jj*ZGJ*_>bLu{1XY+#s!O%8wsx=6q=6y{WSeEl5&-D+ z^I4H(08S@Ri$gB>gv7^dvdouD7fkOpODZed9{wUrky)#SLeM&D_7*w2cB%n1*FG;8 zm}#@>5O)f&aDEzBg=sCXbVR3?o9hZ$mWVD}+uL>2(G5t|Uh&RtY{Fr1Fq9G0XBw>u zSvD?Gu@WXOqt&9Yb}BEixIokmIQ*XYR6nMS)1~OtX%@{Rba;xHkNFK~tAXk~AkIt3 z8Pj)<>q=y*x>6P+p(=p*F6=SAOl$5IUK=&E5@d2cRj=qQK+tBEY=3nwG7+o0NqNkn zU{bEfh1ToFJs{5X>M{u$NwRD-k1(Jm-$u38b84PEYnCV*d??Rics68wAzI@y|G1y( z|EX;YQF?poQ(Js>BXBaL*L^rq&!VVhtp$Est6SmE>wrgJ95<5G*9l5@GogVd z8>Nm8u~eWD>>6c@tT0oiXGO`vv1nP^g}&O@s~z|^-9AtxsL!n4-ShpaMDfmD997A# zwglpm+i_Y_APy7EI6WaZ(qI4cW$~|`_`EGTkdlf|3R1>oR|>Jsgg5}5vu$r?{1Z01 zLt%6r|H~=fyKUtcCYnm40 zoA_om*qxx{Q6+WOkk_MJ-mJD4!(6}C_>+8~aoxgEZ)r~IaiC%D6?|1Dhjyn%+8T~O z_7@g&s~`;Q_q-zsZhnh$mcLxqw$5KAlx^-eeB7Nce%$rd>mWpgTafTt$ZB^B9Vf{3PI>Qo`V#6->dff`p4p6@uVc?BgIYILCwF@DGZyxW~rW zT;n7$c*hg4_{YiEyyIu~Z-m122Kpe7o(V`L7!tm30{p^|SO%~}yro==!<0fFIAY_$ zu_TO8C{KTyqJC=>NHEcVn|H{kV#6j*=OJiOOKA8l^o*NYY_|OOT)}=@10roim!X z(>4|@A1EE>mvQxFs?C^iFFSIqWQAL^Fqr%Mb>f(wPC$a*h!RpFN4zN`MNsrT_u>S! z_b6dL;#;?VbrV<&OYSX2agHGLnkH^{GH?}Z?MbfeA#Vs~)O`okN*$Lao%1&_+Ti&; z$;IzpEO7IBm5bj!^(VMh%z`|7#?~ zH!xA#NGU`FTNsHdox*?_iheH`6a(VMa`wSZ@)Y+$4Ly4QLNM=*qa7Oht}oVXB1n8$ zk!gD8Mn60AS)O;bmJ= zG$rP=Azs^HeO@d7zOU{-qhy8}Ke-}kmun$WN0Wbphj<>lN5aq}E!42Zi4s=1RM0!=W(`s)!{+kc z>#LjZ?W17kr7xOrpM9s6;i7Z%oItaX?sr;qmRzl5_dFDUVVY;8eWsy9vK+85Zu!rq zky(&9UHaUXIzol%p)-R)0WQSOtRW!WIT|BZrThdagh&0MPx0;!X5X4)mW|v%oLTll zr8$QB9P;WxHyYl5_Ed0o_BjMMKUXKAiOh^v`BS-@9E>FO29xN8|AjY-r5l+(jm60g zUK|nL^2{E0wVo!JxMUvu>EP^PB|ZKx#?t`Liv}V4)5$Qd#7)j^{|!A%6C1vt`9uP5 zkpKkN>sP=HfVVQ|?Jdm=C#`Sw;Cpb8)H^FD_nH)SmBBP&NU!@31HH!19FaE ziyg*qTOF&Bv}2R}`+meRR_!f2&jx=5YHbn^+BVyUw2Ha8%wHo<%#0x-v1N!OBC(yr zWM&SF37NxWe#i`qNhl)U<5X?w)wyMyW4!Rc4*qN4+PjvmC+k39{=#9F1A&E8?Rkb7 zBHJSOC+x0H)|E_Clo>~X*y|3PzoV{1nUjX$N9Bq8%EiveAi_Pe{bbr_8+xLl?oQQz z?tGWc&*`7pRYrfmhcSZ7piIjpD}A>F77cfcV6nNEAwq-;hU={a+;uAYUpm4Mg*_^d zp}0~+TIM|=nGw4pA+CQ_fUrkA?awxBxy|ZCEAh;hn^8q_TW_&UmlVli^At&b?aUVK ztaIs{+4jFWjyV%t4f&vd!m&d~0Yr35E%_Z~Xk&MKz1+Ng)Q7zJjQAyJo5MtK+rOrIAt3VOpb3dr`AK06BBf*IkPBb6Md;i{5kqPzxlrz0W$s< zzk;GNyS-!nc&vN>n!zSQ&CHO2xz=Ebxp^_J>8;<#QLMY8fzr_(JKSy1;7vl06+8t* zAr0y?yXa<*CEc_ENLpUK^$2}_t{#7_r@QY+dl4Vl2^XLJQ|?Ie*G7(qMB7)zpV;>9 zggkv@1Qv3#EL*Ow&=E;WUooaD;GgYL z=x>u{J5n?Z7_x7nsl*wTdY(j}E$}?1P)({COHEklh=k#PkQEG=ivg5Vt1?OZPncjlX z^WSU*X{54L=qcaBTu0DHne!>?G-+2(Bu}x{szy)v|bict{ma-1C zJ=HgM)4BRH?6^ENM>R*f!^`#&Cl?>lqx!qtuqijz5F&5CyGw{W5tf{ z0ay~8$6#(|e1A8Y?+u~mfh{WPjR4d+Ie1)5q2ywu=UF0C;kO1wD@ zNK+X)1PUDGsjJuLjV1?XbfFDDKjV!PW1^HsM5#{}r|QiKF+0`%`b6#6P=!r>-%TOP zmRfASjix2#=7$CDF>!}{UmNj#TlbhcVJ6(!pAWD?mKA=lYfSBU2(Bt!wK7c9>M$5g z>8S>$6Zfu90B?bx@h+;0@il&E{tDAQH`E!>d3rqpRv}yW9GBt6gIRW1tO8fz_9oKPJXNnr|_GO!6&( zupAXt?!unxe|JX>jVaLsyvbVs?VP!fLvp+4z0xMToJa+8WnDb_MmOo1%BDlO>ji&} z!=%k~O3w={Gju*eH}$ijBCnt8LK~~nn6L@86t|f9!z*tD$*0Gx1(op-M{Tr%JtsRv zW*H{6az8^ALMfNEMg+~xJ|~nD*V*lT&+9oAgdLiNUaKi=KgfqevB_i)L(6ddJVQXd zA-tUdHa?*QS())X@dFDAFpnwRdw4|D{svtPHjXR{Ri#W<_Vy{gxgfd8D_Iqxwfi*p z_J9zr?U`dKBduIx%UyrO_R##mDO<`vv?&|(+gQr$$QXF8U+_C;bwKqQ*Sxu}f$H~K z%r%all^{;hBW~X;Vxf3$`uQyBow1W)^zL>U(q4I-bl<8Qk#*~FUig`cK2xXO^f*ZM zm3lq5J<@e3Q{7}e#QNUYo>?<$#mf#k@VY@CJp=K^tbzPQ_G9QCJ^L`czSbS}-g)<# zw@+TTUIto~^X@i_FPQfndbL*@ah06df2&kKgnPqcB7uR)&56q>F@DO;2>tH+)B z=GD0}Qhax{PeRNzhe#SC19kS3tJHX@klj;C2I<;V-~iY0#!lv%u+IU(lUIigxD$ zt+kq`#j+jz+h~yMwD(aexw_^tKaDFi!&V#yy%Ech?(h=`=f?H%eMN!okhbaEEI`}T zZcH#Uz^Cs%!#=k30ox392FvOiZIaWLhU_Q^lT+Lxg%_|tp>m&s9=QRG8&udH+wceA z+meQD_3anzb^=3Mzu2+*%H&fKInJE)5pxsb00~kG0UDtULW%it0wTI zkw5!8BQh5gRc<^*OO&`TDY(8*EI#`AzOH|c-kAKYkwz9|=z_kZRxudkR1j`7Gpr@}D*y(XQdGk_bo{=s~k{1pw z>Mh2rsKQlyg;Yr#r`*j2fxj|Bwvq^Et6E7tYyyaCliB6X5i-S}o_6(svryaJy)W%T z9?kCd+gLd2x?lQ41g1gN$c_E}2q0Y=_=gTj{e?~u{~;`S2ht1q`!W%(AwG>Y&(=kL zNkysEjCc6o{*$jlJWnaRmKGIX2b(=j8Qu9oUu9uCFOarM|5LHR2?utqj;=9~Wtv2f zJ1fthFYnF$zyTvWzh&zrt)v7pT?}|)&N)+^F))%*+NG<2OSj=X z0ly!o?mNTm=^D(YM|#A3(3RC_`$^!Cv^IfG@f)_N-iPz64@5LOfGtq_RzXq713CD1 zd=J4eq(MGywUdPiYI~!kVoN^Ep&X9?P<-}you(p43exShPb__*LA`7VJ0-JJvW@6Y ztO!UEO+WYwNG{_t<1|zjE5o)?e3D34kbRv7mF8kwkh`iqw*1mIkY{@ibrG;Ay!z~n zUY3zL9I|kALrbAQ*!}Yl|9v&o|BMNTI)L;SR%(0xDi`1MIp>HGpb}go83AHChp8ep zQ=H5gb2g0?)nx3RH{|g)qX?~0O+imoZyBoIKx--4z*(|as=@dzg|Suaj9`2r)xnzQ>$y$iwHRQ!Ouw5Xv;bW&X>h0FPQOJ^Y~Kd=JzQ#XS80vnXZcJMhsE zbwF0)ngUgKrD%fauTFOXZ3JnQOi*{NqjZY`&C@PN%F-R*{?x-`k*6wP8MPQ(!G=&N z#1@DB5Nh8mh}d5BZQRJ4mkDSNRtu?iS%-L$PAtIk!Rcis-+b<19<M_<(RGHLI&INk)%L;)eJuBpQ?2HmSWoOD*tii)v#}5U^HsFQ`H<76CQevu2YQYHi0hHL0$q z_)50ss)71^(0ZwCh8nMfiiW{kJfOAz>E=8zlebp&7Fn_FO5RMUj{)S)y5LH2WHiAF z*yR|WYFdBAa?UIMyLu2veD6>`_|(M+rdivqNz(VwwtNFM&&%YRq_WHofx-65-m@~3 z?y)~KaGtsl;YkdIzYo_SFZFMZ-q!wfQ-e^|_beO;P4^;~GPZX$!soxBcJ60j6}vdrV`<+ZVeL)Yg#@ zH3dOgZFpKqnQe(qkD-yJnyaG@?`wsdd4zWg6nwjO$>mb> zyVKxp1Uq`Rs!ShEjxT+7uVYcj9;4Iuf$${kx55V`Kk6!dc-NM_1#9DNs6Krf+#{#nYV@z5{t$E8x;sUZd_pU|cmY03Y$bVC3c?-zm zV$$0dkvMdY#bkDmSagnWUuv|D!<)WsxW6me(&p3=!6txej&ukdy(IRw-=bYh#r*VHwz^yG-**EO0-TxYm1a#aHu*_Uvs4B%m~cH_BwO}(o`d^ zWMc3ZI9d6+ePbBlp0p@P&+ryNoLj7|H7)u_R6l&P4$-DH2Q^{DY_m*N*t{`woBU04 zY2)6s#-M-A{@nnx+WqfZ^*;YgHdKD&p3IWbm>np)eXI)bQn~Yq zwN>t?Cp1yZ$?uzL~d8<2sRprgk04Kqa4S-{Ir?0I8@w^+g zXq`R)c167X*@_=1NO+HAy}C6F4KtOPaW$?~`W%Dn8Fj5p<#?Hus7nlF0)<$pH9D z2j2e{s&u0ULlt7e!IVaTCGtUZ{r>DuRdFt{Petr=XbgL25jwJxp+iNb?oXpdbLGR?9`l z0pRlk$l!i>#LzP_vG>>h&*?q)O=V6b)9<8#zdRZ~xU^kC$0f(Ot}T{7Kq+6$AaGD1 zXDE>ukmQ=22`hu(R@*UR#rz~Vyf?rIDW+b6-X7mUPpS&=9ypbo2J{5{1qy^iGDA$_ z95zesuBlKbuBEM&9G4*tr>Gm#Iypb)26$mQ6Y;r zYh}%|g>Us%DcI~ma6NH9XGQtu?VpsK7ZUt+%^Z+|gOk)i`5F+w?qO0B@>JFdfkqGl z`(V_f%2oB_0!yLQ024Y_fmGcz%U<2>aymEUj%qX|apNT>OkmR9h~$h{xyIW4nUc3W zcKr_NUCrRjUwD?WxOv~$Hv$rzf?6wUnkKX5FV{`RKj3w9wxYKjZ|WDZ6@MaXSo{SX ziEqK6SSoMJ+B<^X8>bB0MRrrU$e7+6; zZba|{ht+B{i*mzneKmsyZKfuzDcetVf8z~ZoAOKCVYETGh`0K}Ogh9!x1!4tXx zYs=w^5(#9!4)qgm*;$O;-Lj&~G1-R-JK(oAk|d03<9=2wZfLT8wjjCC%uK@Zb8VR{ zmY{$JR<6Zb?Et3iExlc!@Nab$5F43I9~yO8IkT#mU8kBmtV}D&z?8FT&^89+v7dIfUoM_Z7S?|Z;#W2*cMV?$T6B;cj)Zdax?$#ZX zYXs5s-`xM&zy5n|J;SRabOdFH%90P6iNd124)BMSVLjOP$`|+Anyfcc*$3V!h}Pt0 zeJq7x_Ydn`S{;I{WD;wV$#@EupJEC^jZu6K?P0C77RfY~B8ZCgrIs2EZ((ENpm!m( z{;w`#sJV$|*^x@6I#+Vh7#!M9*xne*TQ!c{pIyvSQBP{V_NDGz<6*=I4iIco?n3ii z5v1q3CcE~I5Wxi&K&fZK6`H%_B9?az9J1S_lgNlBE2&K@nEHdNZp@&qCa`|^TOw&( zmU|X5cDvUrnJ2N`vXu?mRADF4!DwD3&lC&F0#*i>3k-I&+RYmepl8h^^^J|%!Fl1jiZeo*eZ%d_7bTK=P!|n}!EV=j zLf({xx1&)y^G^X5G7S(VQ*648EbTV53(MtG=G(eXtM`#8{CTgGn|mBAA>RwYEx}?k zB|?^<30i7C;y~N2CfHbixGqb`LVu%V-jLzl4@-`)F(^$y({cUY6RaZm|7Y*tnimA1-%qF0Y3F1swbEa27>vGoQW@I-&uY~*PpkUH!ti2kCUVsKZ5VQT2x{6K zj#ofkS2E#Ihl3;RptLmS;3u7rQ7(|0p;|@9s@fZ;m$FSUwwWDa{cT}|P>};h5JM_~AL7a8bz@?$A6}{73+$11qw$8E@OIc2n zdiSRG`AehWOpL=frCFD`QV#{}@-c4_wei7gsj?Z6XWxkrQWo#Hg4_QYwDRmCxup4#-uVf*Xv+6 z8!hN9Qyxrf7cXe|bK7)3d>$!p1WI}@;cV9&WGz>gNh-vfnx}Nh2rV4p6|xrH!n>U3v4nUaH>~e3Uz6wLRFCTnSi^#HfRt7R z&71S#3&IO%qG$L(_eADoK@-I$nim4AtCAv}Jm6sZfAzPY>Zq7pk=wOHg7T4zGUY?4zE%XDK5dW#s1Rvmp+8#k63yM)sY8r^{!yA-?z zjXpFr23XzLU0;=h8lim)7FVv^hN1};L#i6Ocb}2Rr=6Y5PPq&=6Mm41sfJ+(uRngk zv5zpB-56_KHFmI*J>1yK&P#A(?E1c}j~(u*hZ_5Ufl9F`l4-UFLp|DFptUEXe1Ds* zMtQHgC`Gsk--Q0qrg^|~&@*OO1|y@zYGQ&d@0j%<-J(KpqWl6umMALk!!U+s!;;@- zy~lUne|KZwgKD;7Z7>D&Cy(F4-&~vB4E%j{Xxa_@`y#q9qkq%nKca)C{eQEC!6kbz zCxCCa2jkFAGVX;76|jYbD$!Q2`hw>W(gbQ0s9aot@LBE01oqnmPIg=FYkPR2r97U} zd`-T-e2r40wMm}xMI}3)VelaOCsGDwvR1s)XxJ#K>?!l0%UMK&lE>Y;VpCY}=t@X0 zTC`S%$;LHk_TdTR)fIHa*!8s&cqU<=z90)mlYGSCDbo$-KrUAY#Ld=yB0IvF!vy?<_HB83-b9&~H=<7tz?! zk8S0&DOXuOdqz^eTouk%me5jkhqfP|1nT%PFdOHoq2yL?TS79L)07pMp@!q=Xf7mq zW(YB=9>He94ybL~z*G0(zl0}x1P|i}P|Ei|j^Px64)7NIBTS2m~{yU+u^%UQ;`AW*$o(4|?aJ>u-PxX|}2p5SfnP^JB z8ql^rL^c>$*|Px6xufl2 zr->PrT}x&ApsoL^nj^8k*;3eNzVI1=r_ngi+=>2@m`{7-S*zzFN$~2plJSZ1Z7*OH zfG_|TVC)3V+8Fy7y?#zH#73WT4{{J%rX=N`b^m^Q-WW7v5>iat|J)hGK)|Evr1*(m zJHH7{HF*gW6Yz)kthkr?Eza3Dt84WpF9*q^A7ak=$41b5FNJ`w%(SN&vb~P><2xe_ z|CxgT*xiRl+WlxPk;L13(#SDfqGV^AYR?}He8oi+$x;;UL=zGq-fX3fTh`yQme`7r zS{fcq2!ap?p$==b0ENDW6>lb2?Os4!0p0SE(0(F4py|4pz8SFWOU$9899~n`oKdvf7l2k|xQTfM`4;mqL$B({)#OEltA!7>B@Mqcgx)nxwce1=kp% zca$ekG!a1Jtb4R?tTS>W$xkKA*JPQbZx8{Aa zRIlxQ&I{&>D^n6NQRFo7VyD}-nhgr(KmQ4_TmQsHb@TA4R!)Z?5jXMDyl}yU++@B-JUpyG+ME8GAS>kD_PBC7bN zQ)i&0!L&p}P=t=f6x!G`JE`1EO=z>$y3ElZ0H`50e~kakNMbvLU=lDAM0hF7 zoTRYh0rYbNGVW)dmZ@Q#Qq;e$26t~Gu0Sy60zSf!5`*sBr%h?r<<_YRLBT4jMvzry z3i2}c4>LsUo^g0hWb?=gp488rHM6*7EQQHRc*t|&!FGQ9{V&rBa`W`#4F2Pbv%jAG zV*l~`v$NCFzkl=Y>zk`L|KMl8{9TmkE&l75)4#Jf{|Id4?Ch_bV4n7=iSt|(Q<+ck zfDOL*Pq>vG)-e-wp4e*GUo*jS9yMl17L01?g)C{&$se|) zPPZFVcRMd}5Yvr6ZFya{NEZ9JyMOdzzX5>_v@tL#Q4vpRA_XH?KfH2|4TwgrK&u85 zL?w^`G7jX_twg8w|8&kEKgvU5>uDX3sjQN5^~0+P;WIX~|8kAQEMW!AZHZ-rfr|Q@ zhC~?{z8~o3xBBVNdSZYCJN>fyZ8mt8of&6Q60CC2T?pDM=#5 zvIE=PJ{JPLYWlR|lEQw_0+#?$d$7MIoL^6?FwrlBEWU8m?4><{ZhI$rOiEDbsUW35 z_eEc$W~1h7l5(XrfSG26$c)@-DhvXiCo;NaF+xud+~TfpSOBo-TBF0lz@Y&mrUeni zw|XUTDvB1yok^1qqFY!|6qK_V8Z`yk2TxdqXm28U^{gfQ+x!Eaq%-nfYtTv3TjN@D z0HXnzqz%C$rWDQB&dZw8wVBAGOdz47Z|c@;1DWTl$jhieqxd|Rw@i>IDOJJpexj_` z%+DE(rwF45;o}CEQn}-w(iq6%8+SMK%~`rFpo@a#P=lQ+GKXRgebz7gd~kL9!B}M% zj<;0jvQLPojkw_Jerx|2uM9?K8%YzLa>{rWg)FE+YtU!hu4N=MUPBB2;nhG7Sma(W z9GIXAO?f66A1nJ~uC2)wsQbPDg<%=!IT`4u;-mfUw7|mcsyB0pj9zHaV@&!Oa(m11 zuC(l=3MPslDyabt*h~~WuWWR^T!jE)5bRDV@CelR$)qf^qQW~hH`rjig^^(ll&Z@Q zQiK}qq;(Plt>V5bX>KZg6Zdaj*fD{oBh9AX$$0qHu^ZWFzVbgHR=NUB7hfPLyx~F0 z8fLnpcLP(!XG;M$bDLqs*x%uRrZAvsTltCl6W~?EC(yU(zdR%x;j@*W(LS0>qNsX^ z-LG1&avFW99FO_hcosrd@r#}~Fv(2;HKLHKYU+)4Z?(Pm` z!`AZ~($=MeVMlEPJQ!e}`>Zv4iiV6_CJD=7p)7K3vGOx>z^8#hO8w4ve4MAoDnNqW zI{^@(a7A0DWP6VxNAX8}jp-IH#2T0AwMPb?vI@$PR&cch;E{?ON#iOlr1#bbwg{QN zz9nlPg|&`%aqYudm5gG<_@Jb&#_rb|sV6r#`m5Y>9b+8sHT)KUrOJ2@@^14P&kXc) zadnA^Cq_F&SP*-^yWl$ng?18p&;k=&WWR?7@Nu20zu_WwPErRqk~7@`jw`g+w{_g4 z70=^oMgeSKoiR13jmesL!x|rTX%Ij~+G6`)DXf9ocsJ^M5x5f7jvZolLl6|zvk6Ag zNbq}fvm1*0sh%n9a=>c?whw=hgEBroW8ycOPmQ-7R@TtW)1GtOb|V;o5N}L&l4LR~ z6N|U(3K?Z4RZ}W7VSdYR?Fk9h*|CW zo3x!c(n2P*ISWFU-}YT)EZ0y$RFAep~?pV$}|s7Lx=J7w%;R z0)z!IQ}v(vw^f}WA@6>fmz+s=I&q=lk;!6>3Z-CiWw%InwH>Tm#Z(O8k% zh+gGVDF@E5*TFq$d`}u*$MdA|J;vDq3pnKYd+*CGbzZfxDr4vC$FYMbNjvRZ6xi84 zUC#s}N^!BYX4wz6*ROLetQcLuJ302Sn&A~*FNh`x75Aw3`-w7MUj6qdVp-ug5V0A# zW++WJI*(;k0?@E?JjJY_JPEpwh<7>MbX-RVWd7tgXmqdydJyN7Yuu0V^+On|7q<6I zT`tH61_|=G*>65)RA_bdE#-;cYP;o?H5^i|>)u|^S>n#DpR^OCanl0Y#6YxSMTHhV z@3<_L4c078-Uz}c&ly?q<;uPs(Tqk~@y%Hza|e(dqOhrg7Nx4BBe;6OdwW#hp4-no z3sM-58pg)QWV~&B+aKm!e?o(v+rJ9F#mM6fPOU#%g{n56?JnSNF;*Ero*PzT1FNe* z@JOavSul^~n4=$h@r*nrHDHEw#G;tG&ZkvO_4$ZaXtM+dWHD9>6PWCxVc*KijO$O^IG#^kS?>WLt zQbBlX%U$S&o7$QP#^&qBYXG9zXRyHIkbZ;QV#Ld;6Y!|{)Cki7Y9K}mAZ}nvDrCd; z1-LhcDmBe09csqhtf{W&qTt`D}4Jhk@QSpKS?@91;|#`RpGgr}=EP z-oqd|&1aX@J~+ru^Eo2uPH@+I1>tEvfuKCiXB0y>^V^2p#)a5c&rZR%_6fJU;nN5D zv-$1-2vGAGskx0`xSf}55Jaf??4mD@rl#%K(9uAMHak8NC{gn{E=W=HIh4k+TTq-a zRlZ6n`UDPPS)Ke(?)tA0m&C54l6c&Q5YB~brn5oUSiPobx-D*Kz@5ZjvT9A|Um@sT+(f${5ghz?*n$QmaFdSoNF2 zZgl=R5W}_egL0<5mHG!BRy3E&g`!J6S)ZlgD8U2JtbYr~R1yt%%qWFy#p6I@pr znM3wm?+OKfrov2|Lq@+d>Zw!poJ6l-NSxQ`~$($wZ4iR&DR)-4L)(19})*a?F@<34f z@B3{mbHNwzIHByq18zY-KxtrhN-ccdvbEI)X$G38!xr+76t%EEgPv*7e_X16A!P!r z5?ThOEN4Z|*&QP^pYtN8fU}4@3-+(I$qgGaNrC4Y@E&!mRW)3jXksKmR6;q_ zoT_%Mp;RoNE=wM>bB`!?mz#*e-1*gz*E_W58U}sK*2;6)IswMF1<8C#^=$|5MYLc` zne(5A2!@tImux{Xi}_Xxg5$0h3wZVq$#B)>xxi& z$%3H>?pMsI!9=l2oTHiM+d0jw!9bm^5>8hCS*gg~|+J`PpS7o6qlcd(-YbU45+0*JLAH7lG*eFe;7Dn%=r z)2LuMR|Ss{C)cV@hHg+b1FT%V{VUm<8@)fve$)5|N_h8*2Z|q|7*^xX{xPtt9t5{4 z$jLs^?lJ5S-r5fwicvLgclV1;bwpgMePL1^ACKy|SX7S&hw4#aP#qh8Y6Jpda2E1- ztNQ`7kB^^ue4wo35oZ0cY~zuc#$&QhM~pQrcX(vp@Il5J77iAWf*Fv0864WOZCJ|) z9FcAB>j%JhIu^FmesP_SiRpA4Jf{&@PRGP?IvR%4Vey*|j@@(|+@^hEHXRDD>3CR8 z$HZwm21e5{@tKZ;%@i!d9}$yjG#=9fU@<*X9HuAD=SlN<(tJjYH4p=7{24eL_R$mW zk^ej(=FzWN$qvLe8b|jB$258rct#I}WwbSp(Xla%!iA7f?4oUx>Vg4S#)Gen23~p8 zpes8Au8hARkAY3}a7ZJgVCF*ch@OtYZ|@i!32*4Zv4)O}GxRGs`JpB5f^dblz!VzJ zWf_kr6n^!O#}OKfA>@`JfLw<0gF+L&gFz&=$=!c&G@#v(fX1Q#jX(ejNB>hnN|KF3G%*#gODB#O@#2tMP{dv=DQ z8H3srir6y>t!E1ilOt3R9fi)bt@bpELdLU?2SMX`5G0<5L*d!COul`H(m2$eany7i zw4LK3?F1gaEfIFct?z>&>pT>y&ch<=JPtIS$AF~sU?@5VM9_I0=s6FHoD;TC$41O~ z2(+A0Tz(iOXB)PDFm#+lBIAr&w~^Oj7#dE1=(jTx&fd}<;m`&qKAya0SeUCp;oI@@}j$%8a1N}6=$4ptk3fI2IBDDL#%Du+-D20%y_@&a#Q$^@T_yw5p_cuZZyz&Td0@wJ7N4nX%-ec0R;&a%R(2KZ=jV7vy^}&2qNj zpApbR!OtIysnr9>5Ou;0YeQv>`ii7D!yaChs*rwAQCg}Y<5a0wm+R4u7M9S7b043v z&s2Xgi)4D9uBQ=n>%4;>pM4Y;8O?`!#^jv2ymRJ`DME9&c%o9cA;=u!p$3|)aauEe zXvsS6So~2qZMjy^Ki-nWz| zEZ$+PT5DcFBf87Y320tBw8ErddCCP%Fd@JU^}UQ44$|m1`&m&KFc(~{SX^7zAW0jv z=0%wU%XVMO1rDu}wgqGHpozjHZ3`K*jXAdylAhR*+N-Cn50}1Ly2)pw!yPF>w_c|O zI|hUOWcPSTuLKGpH#rpwAY0%Etv#D0A{=0I+_R73?VvBf@WPi`*JqzIiW&h0aZz7M zslK5d*PeppJj57%1cWLbtBYM^chL;*&RNGD`}Y%b#hmZg7I5^mUZAY|^prswMjrQbpLHx04qQqoHAc=)3^vA*GLIW+tYoK0uK1#WekY z7d}6QuAJblj}nTvgVwoXZ5>c;6)AEW-Kv^Je^6nxtr!FR53h^~ zRMR_$&`&(ePm1l6VtYu%w#vrIWUlEE(&K}WRTG*oZB~zGUv-;n67-Bt!z|w((rK)y z@lkM9f##un?x^giQWkV#f*kWk?!?EP2(WB`<4_HPZkpVL$1=$lOax3*bT(Afv}@`^D2CXO9!Dg2dA&IrK>mZOk-G z4z&yXBtiZPB}k6}*V2Rye+#`K%-JeM%<=@##ElfS7Xp(4X=ksV*{W6RRFBFWLZx2W zzvyXIzBl4K-)0yIzSdQSr2LJ#i=(@TcOY*pKe{WE@~Rna-)`a zwIFG-=ON4)ZqP@uekoImF2^sL`OdD?s!XYv=8VQrd%^rQXx+Go0gqoS8z)Kci?@~` z%Qb51Iisqt3Q@<*15D*OzB7Ld!H$4k%d&Cli=KV~=!bwmN4GMS(PL|jva4MzQ6ijEm z-vx}eD<&(YS)Xi&GFNL1>%Nl)hWcvCfqt?^dyuV-&I8lPU`Doy@hSyaVbCxEc9~}D z*my0>y(_qSphHLn2-PTi1Bv$%{kj>qz0du9w;2`Q&}xk2VIa2T!V4hrKd!7p;n2De z>%XzF?X8*{Kx>z27EsFMSi2Z5hKF7cDJyE}b(0Y6Vf|1A!>vOvw$p>l z+|XPra7nxn0s&?C>_6KUCMc|n;N_0T!7jw?qkzs9jDoRGcUJu?q{YqASo89V>fRvA z=Uq)bv1?7ofekxfH z*c!<=VXUC}l5HzBkS1dqXw5LJfW--y7>taOgk*kml90-tT14Y$rMb#Lg z4&HBG<5ob!KXxet$H-*qeWB-%NO6Fnw7D3=QqN8Gxl5Fy2;t|#Fw1n#asa-Fa++y5 z5dpj%LQ4>lH6k{RKmnU0(82Aw!`ls zO(IDeeNQmA%KI?(R(-QOYoFqx_~RcsNTJcsz+@I@blc&Al3 zM8>{ZF(?rSlJ1S`6J*k&pwY^xDfkzL(C-Dsn!wZ*fM`ELD@KZ(X9**Jc327%wpg&J z_zS946{-S+CtK`-KimKMOWW=b3A5Nk7b^p28A6}qV3jR2E!C?DO6)&?8pVMV?9sietFvm=L2h(C276a07s=j5Iz zaYSz;dFV=n2bjDq=82E;E`JJSr2K+X}j!l{f~ssG`VgErG?IT5srH z-836)iZ>a{xkOxH<96|n#VJ-cGL+ldi(FU+M}DjYWMdFSeCBa*cRpw2Z)=NO z(5=%TfPxD#GGjD{$Lvyd zVIX1m0R|fo4>Qis@PN>-+k(D^TlRy1UJnEDItJWzgsm_f&~& zUTrMqebjW59xIkVaV*d%7tTB1T#DW)cXncvbo|-i`atRJtZM}aB!PhYVff$w>Rd> z`0IOkGq_?tY0CXxC|Yo};Cki%*K7G+7S`eNX<}@mel4v`Yxz*^rz-ELZJ3gcBGQ`D*&J6pTbksS1`)99baHuO@^Sc#p$G95i!p z{qnlW%$N|VB43_=dH$mNKwX+3!H6n0Y1tV)rWf6F`-SM4yfFX!rTO2l$S*x#kSlUk z`(OP(Jzq@qrg%&AVp3;QwB@%<^Sxyu<=_&79diX7G9jf z)XvB~m?=|wi?*-}$V1bEI{j+;;*3a9d%d%pJ1+HwXlHc#qLUYHPW{W4sn=!=qo!gF zLmAH}EQ`1h>wCI(jKaz%OMd5yRAxDgSlr7d!0!&{p4*5w>}=hOsCZeNAU?9e(S<9? z`Se`Xa`FO>$aCx68L0T@c~HLka%&Gt>dyPlSAlEEbJ5cluGVBg1G_rFy?~iGM(Nn{ ztr3AM&Q{H|+p=&aV`9YsV&nkucPa9$&PJYr|H~;S1~R@%(${0PM4qu+Z&u$ucOeWW z@(#0wWOy*6EWt&WhtC^0Ceq=GLNv8;XiDL6k)Gij{O>XWhve`)M(>Q}2sydO7$qBQ zks5GRVMQ9_6p%)77$maGMRfuNPbN0s&^TkroUP~`WSRr-@~GV659sS{$ieT$j+o=9 z>f7{u;22b6lm(9S$0oOP)~}j$)F&%%bCZ8LX)J>I{onU_!GGC(!SCgU#{caV(kkk0 zCE9;KtybX-A}KaFgqG_(XL%0^8kt4$q6F?SA%fk*g;-n4#*%sJoxLx+573-Nv{cL% z)QpMJ+iKO?uTH-rQ}X;wleoY$1Y7G%A12Blgy5By@AbO(99eEc9~<()Vec~`V|KAv zaKVf9l}vcFa}d@0jTXK8Z6)u?J;QmzUh9>E${5v0=lUOmrU~Gh?gBC5P}761_iRSK z%PP{Xa>}xQ09J zW?|q@aJQGw!+~*gv?SK8m-?@;IwXS)?bFtLW5P6HcPw$10b66lvj*svHNjl!*(fdr zQcRzHVUf^nBRr7YZy=&#f5h&$*GuZUa~liRB;U>+_C?l6IhsAnvM{5V0DRKYFirgv zi_MD*X>EEv!0L#2qD8f4hwjUI&stEkz5Z_^`ZLZ1;Jl5v9RL$BEr&Lv1I56L0e9bmdrLo8T;;mnjS_@ zt*d|(EJ;jau2XBZBoA3R=IuU%(v;NgTCENGjTtwe&D900KYgS1_(bO=;C}!*d(%MH zW(Y%L;)*31$yr=RLl`!yM-?FC!q>ndPr$LwXSF?khB-wd>`LbgIN1nuw;=3}j2g~Udr42=0J_uMtu}%+3~czkAYY_1XJ2^D9EX|r z1q@BCU9RqD5=RGd4MenYp;%OQnPO@axLlB!E3ggNyA7BDn2JgTm~+sU zW|D$M-Dp2Jrn!^}Yt2PeT1w(#`8KD}O2<}pW7&ov*aqO%)~0{WY%*ak=XA~!UaSc< zwNHTPD|k_m@;a~CcgBB1$s&=oz<{~uq*&#$T&}vDOpgpYcMHJWf~QQDA&ljniSBXP znt0LJg5dYmfDEA7f)<_OjQHK)fvgeBjgtoDv(BT|)K*>7ypVV}J%S~QXe zOd&FHx>YFbUQ<{^AR-VLLSB+*!$?KrlU?3>$UvrObHF5xEbhY9F#v+!v1IM^8&t{+ zc-WjqxA!!U0mLAq1vi|35OtoIGBEef25eJfjE&R)wQ+^TxY1r^3;mlY4O3|$sH;?} z0?rTRT9mxc&e7UirH+XkP8xTrrE zxJ4e%Jz*FrPSZ=E(6Z?aMb-df0ihQ0txQX|tj8p3pQoGW>E?O5d4AJ3k70*+?F_PW zeE!>CI|0o{^eyl9sW!pmX|b{qBd;#t#M;0F4+q6uS-oo7*lR3+==5kp^-VN9dAcG5 zKOkHH9~yXFq^1GH_n6 zG2X$VLHG(MTt@@?WC-Qlvh{8Yjp$2{Vx4c<`g%a184zaL<~6Ju4~{)d)EN|W4kY4i zA>IUX+hMULRHPXv&TJ#f97~Kjk_fY%_;Lu*<)OruM=i1h9-uH$WxSXILdY;%2oX`j z#FJssq+cuv5=piZM=&C&zr<8+eZM!c#~4%u%WG4v_IR1~&bNCpr!IVU<4-*;n(9B> zM5h~e0419$6F%0ridmM(wMFf7h>K8C3@a=OjJoBmHGd09mP}NI2E6vK3tDK#bj=F# zdlwUrh$#Bb?g)dwTwpzes^&@!BbF*vrp)3V7fcnT%v6yxn%X65&+qd_M(-$3KxVV$ zPoNUH#VV3b*3VJ!Xj9reD)%Su_S5^t5|!V6Wdebc`5 zwN_oJsw`u|K-c4aQ+v$_lqZ>6-jkM=ohUgmMMiSw=qASOO@xdAteisTbjdW2v}K{T zEv)1X)x)KuB-Ot_Rig!=<}J9$J5bbUHG^%69zl0nEZ;S+5kz|&r`>NOqSM05X*Wf$ ztKtJGg5D2_n)=|GCdtXmldbbPalX0C=#qBFW`|F^*|CVfc!k z+r6PvafL&+g&vsGCG1(=dK9@?$ej_;OqB`U-qhxp7J11g_|tvP3+5GjIbG01H5b1J zJkKuz1)uiMxKHxSn?$th+&8Jx&PX|dw^|o-b}L@Bq9X3g+;rnj)&nRR{;7aPn)n5n z47yp$C|DKr(YFV|+k+8Z;PxU5@ej1kU9IKCz_cxfS4!h+B@#Gy;Uw+|0>#nAht}D$7R_;WewH#nNE->J~Hv}${Q0A;ma(@1Y$Y!4Zv@nbd`@B&4e?+lR&t~K@#d79asuT;BS1@e&ftEFEdO%-q$D<0f zlHfL3$$Jt@a?ePXvpaoVf>i2^a^BLy4GvHfGUZ-zZ#W5kUegd42w=PQ*?GuFLUavU zTJ(MpyHerhf=Da}F)_osO*tx!ti%V5C$&^ zu&I!d)M4ARrFrwxpeigR-VLmJMX{nV_&as`cV7p`JvDg9Xr^5$>XrlDSPdYBdMWm-e7&Z%3VUXi`W*GYda zbEc)1AOXp}-4uQF@8mTtC`v%co?2aTW?c5V)Y8oy>gU2I!?I0?yb-~Lv3w`vaG~SG z`^fV1|I#CTg};+jw+2Rs{yL8z7>&tyV3vaa{4wVRo3*){++u{>UYf%DUCWIL*$?bw z{6H26w1zW-7gg_s=82pWL+ksP#rGdBM@{1U50`ch`ak7jWB4${-Z|`P_=gf?-!`Pn zFcHB6-)VfmAJKGn1feu&*>Ab%4MeRUWc{{QmRM9lw}Eziy`q|zlC(?;p0y(LRS!_T zrZK37VhKp=tiY(qYli02_@%8_IY@*+v8|20!!qrL!MdvLn$Cw?JVl$caG#mL0SqEj ztT>z1?4}BqVP}SPFnMXiX+%!--_F4qbvCP<84$i!iB!F<>qGEi2XQMz{29KjEl=)! z;rkDlJKg4eF}RzzxqiF#Hiti4v;2-n#$4LDJjVKUxC{MYv=;wbY0c(ajS>l}Pwa&X ziX&9kHZi|`N&4PNB9R!oBDpY}4BFNt&<>AUpWy(50X{Wue)bDHJ!GfwGwC^ZfkPTT z-_-(lzhcL@?HP|)wpxVMKGUzhy{?;TU$JaO-d;Nqy*jyi@B24Hp<(w#k%TW;!8<1r za;8%$cpFmWoYzUz}_}0rtR1--Fg2&wAXMWJ4AMPaRjLQVLB> z%qvxuFP+8Uz3Nx0a%uvQsj(TBNrL-V)83@m27GY@hHWdsn)W%Y!d&Xr_5c7It_J_Sg*U?eedWmwkP--0N*KuMd-V z{cu^=k2-;Aq+m0cOuStxv1Z}>VurdceTJkj_T=_*$!RPVzN2@q#^z|8_LN8`kLL;* zR)z*7>tRKQi@1ABLbn53sp~-IH{+#r(HYcQEuf0GjmM~mV~h&9Xzok8RzD?E?0uTW z)yC|gES%GlFp9D!+jjqL+qP}nwr$(CZQHhO+qNfWuM?4PP?uk3iQ+J|9y+9#;ISJT zuE~(V*M@RS7Oe%WaY}EnCv$7>0LRNiQpwe{M^Vp*!Ht{CDyi0YSAN-D%0Jr$5bv2t zYojrD-e_HtVZ2I&`Xn0oMKN9v4BAm+4MH`L3Rg$IiNt=Z*DVf-Ex;U>ug*Ne8P(IK zbmk2}8#$sdW~B6hVmFL4O6!9&EMgZb5HJA`AF*ik0W1AXWuvX!l&T9brj}ttMGLU| z7%w{Vhb})BTF}bB{u34rp)r;|t!_7yjz62r;{#{-`|$f7^Fzo$mNZ0taDEbTzuMDt z{zu<9_t)e8Mibm%7DDKxQ7v8fd89dHDfs*L;&w+vQ8}tNaD0+l(kHiP`~4l~_de9W z#U1vUv7L5T=E6bho&tBq1h-dwsDEC#yk>V77!M1aHe zFCVW=1IFFSkd*+k&&0T(ch)3BMtieK!YaIcp2tWGMW&*+bM6Zc6UTWonuC5`U5JN) zCEm^oWzBjW8DzVlUI{I^5HLRl>SVbz%ednk4GHky!s|MhaUy_c3WbKwhnm+{yNPxlNzRunKBSsa?oFrO236hA_u`m;EVvc_^oY z7HXVvMkb07)S9j9NUd$(wA48N{>VjZP*UquLIcSRkdfrmHfq;zC$J6wNm6CFa&O3$ zL%K}X^G>iF0DttR4M;}d3?o^JT-_h9Jqjp$Ro2RUlg#M*%+~s?=G1Yq8B1>L#Vebh zUf%tB;Fuc}Mn;@=LoG1W;Ev2puh%Yzx9W*_8L(BHJR-fickW@5C^@e2IF9CCccTt- zK7Y{{?AP1jGkrF4-hwxBUV}CYULi!V_h>$RR#ZT=etOYkALzDv{UZ>>ZN`9nCj)gV zTv$#uMha+a@98sMv(SKsU;*C;1TI_FV+Y4}w1k_`7NqIPS5QQ*kdD{A6a&d80bsq* zY6FARdIn>zT_9KNt2G5`y)E^gSGxyKbgMO@R%cL$Fog1nO@Vs6c9`KBDMtTzJh&*F zLu?3r-lH0qghPW?!jXv!sx)-FBO5xAfk)7o^$a^9!;Q!i>N1J8tRZhZrX|?Ogw^jE z38HJK0mmM+y8kmssT@DqR+A$2CI8$^*FZ7y^UHcRa`RcdwxHt@RGS7x`cdS2F>3NA zx)Xtd$@ZKhGxgE@oV+Yj&39XE^Wpj zW?H3rC#QM8&LWMKJ;cFO3f%47Jiqu~u!?QN8q}r|UwP&d=c%PUjm82~wZ_wRQl5gV zR$h90LTqe|#vetewbRz@`>`ngfd{MsOM=3tQJ!3udx!}IipSFj!M|K~Qjvk?y{){% z@`kIjJlE|-Rlj-(CmHj9R7x?t+VbCkR->z4;%hHu>L5bU3sabduFL%(iCzg(CQ1(b zAP7TsZ6eb=t;2Iib^kwp$OQYw)~T1r*nb1-Yc*~!v5fD@OzPG|-33_t3c$ND;~WUT1To!9NYsi1u^ zE%YsQP(x?2Z63YvJ}2_&+PP?U_!J-MopzAe=7ajfNKU){{vEq3@u>^k`vk!k~8 z8pKqn$It(cU|g+x>ohPa+h=|}|AbIjMiI_*RA(c`I4WjDD`amcMg7~voHx`u^;Rrd z(-jIJk4Cs3L}29DN9hL-#-t1$zLJv>7d)d@Ku$sfbhH#DS|*j^)!vWo?{HExr@MyZ zJ*Wb$;UeHYz9Drq?J*3CRHZJt&$D|ubn!20<Mmom0?gxX#gNEysQvp*tb5K#MSRKkfj9*Tqbl!3#uYa#DRKD z1O~&h#xkaNX#MC!K%75OiXTvl7x;79#|OUK>|!hc$!Y)rv^w=7vK}AE*2vfv|7OkX z4@L^sF7@DPyfS9x8d$HDj3AF1d6o-~5FagIE*?v-0s<+T+}=Fo>)ny9hBWj}=|7q$ohf90zDy%uTkoSvyiW#hRD{tEs>pAAEPLOeo}7%Z zFidS!KD~;t68&pft@Qv#JS$?H%OFo|Aw#&)z;#T*9lWT>{miJ;&eH#JeWSToNE+M3 z6Vu?CjjHZ5IgH(2=x&)^J7rP?iTuD(Yh2UP;8CvbgU%NLVjM=tRsAD%zpoL&?VwN% zgLn;nz+8mW_KlLeh3F%Q>qW0+mWyXXqei2f_~ZI{|I#Zp^W)+9`FgtW`#pLq!!x_b z)64#Oxkmen+sn(z$^HGwa?{(>?)d=c`#QNRl(u`&>&yPl>G=@TCo(hh`@s_!s**7J zU%KaTp$9Z-$4}%M(8@u#f}>n8tp)iBP|Pd(-2&`My=Z6dy;+CAX(f9^TsHk;aZ zv&@tPm1@dZ_K`U}df3vsx~8gJAGAW^e=yP=#Q-)C+mN1ivCBV}&*Il|TVUv4A*?@S zGi}xS4HILlK;k&XVGLV+(8uRx3WTpH8q!24lphtn5#e}e*ft=l^PHNgzo4`;V~vh4p#BTh(^$^KXg zDZ{UBcg+Fk0B8BQ$WxdMNxCd-unZHIUyVrK!VfIK>;f!;#rxDOEm83dQBv-w2YbJ( zY7E5BPP<=}Nw_zcr53H-wNhCvcz(l{Sxzb7!aziS*sjelAC{z_LZ!`gNe`8~ELT*6 zeu&!ij}-2qU~aA`xw z90!A%Q~0ExquGbRC!f>j@)? zxhRqD^C{6kju_j5Wz*Sdh^{c?oD2|XzoM&(z!A!{6GzYuyLZ)O`H+c- z9k}yV*BHz?@orS+lZ2|K#3`=YcIH^Z^E$i8WbKYcjPn-H6#3jlr4e)wqYIor#~4Yl zOqQ>rvhR7p3rap{;C9QI#@sMmS%*7*k!{}P-Ojj7!Ywj#Cy2`?mcsOK6O^lkjop!z zlE)iFcPN<`#CIXWk;O1F7qj*z8J{nR@&S*rqeN2oyXU!!l^2rN(X=W<&$<5Qs6_se z;%oE|(RL;^dy$vaJ?2xB>;oPh;L--jni`SHbwQ-{XF=CJu8N*D7>nnbM<8~7VNmgA z{m?aHATb}ONFVU=D%d04F)xjUe^M8BNHxA{h%2yCO~I&M(EmObn4Y|N4Q6=D!^@Z# z{&~L)Q6=R?g7dK>T#r7w9#EQCIyAgY17DtIk2I*I+K*!s-+@lixM=^}TME8M&s_Iu zGrjgw93vs~Sl4RynQ4tOWe3^`y>0zGoAGe`to2F%U9XIOLJL#Df%y4adLNjtv&p>0v+8O4wys!5GDC;+D zWQ!=46Vef_Q%Ih6bej}T#$1psdaiq*5LAJJ5&Z8THFhk>>Sb=-1e@&2d0vNemH?V& z{2A!=r1liusvH~rAUy3i&&#(*AlcXWKXDBKBhC2(>i~z}8n$1)pq=y{N>JR`&)S1lT8)Crb@}cGAWhn6kYFO< zkICC&hRSAH<9eN9V3L`c(!0BY80@2-;m4snW9-vQ>^rJj7X=b@gWOD@Ae>$R1H{Tu zh#+e>k*+F-wLF>{v6OUf7Zn%-aV7M{XA%B0K!T)KjEf4Gr}A~Psczpdb6-!K>$I&< z%u3wp6%#=t?fqm4UsG+Tp?Z{`Lee^nhaC-W`j(sJOny_-Et*~3pO+>YP7sbJ9U*wj zo5kTMO&-xJIF5HhYgES+J&-9AX?)mrGIEIU-LZOoKr0rf$rSxJ+2mpf@E3GaBEA8?%T!DOrDC!sc=0lQm?uEr5F=kupn5(woY&p zRo>mb7I-t2>{k3HPwmmVaX9JOwJ$?$dBckSytIyhw_Risikhd??*w;-6 zuaU^J2j(J=22rB9VphtuNv!qnbNuvz!o**tElG8xt2CzA-OpN(I&pk$y_e`*6LHxF zAIqlasd5dgDWV(`_jss�T7_g1^K3oe?~1eda0Kh3(>5PccO>PI&@?XlpqeacL1TG z*ID|)?jav~cp+fo6`Q*(3T{#>5m;KonUModVBk>7A_&DYG+sYQ{SY!g1jIt&9z1#sb)*ReHF%4d(kSwVcA{2zt+s7lDbP77isRIZF z{r9M(hcxp6`xq$L3H>x}qh2ys>M2J3{YjTqlLjati}7J*^a2zUXy(Hac)9%}tb|#7 zc}jKxtil~kWP=o^mi3kNzM|64pfu1Ki-BVLbw|*1PN|)N+O)$!ch2#o)0lDWq&IesYl5*0l-*1w@QoobPo5OSyV5Gp0N=ia(S4eKIE=Ou zHO>Rk+F9{MqGws^91K+Cc!5bzcwQ0P&f41GJ;dX7o6QG{y>KR}H6zH+Yu3|@n?+a6FZQ;1q4;T(pd|X39s17*X(eul9ok9#7@VRH| z+rhCnD3#v6t;q)cK^Xcmj8BY1t{bmWMMfb-&o;09)0jio)TsN(Y;nKjR5T&(jF?$#NN*tWXR{Jc_FcXtr3_(`R9x*pgMagXET-8Grh|k_M z=D2!Hu8>U-(u?J(ayJYQ?L)_qKzRB1{cCOzST>{>2<}V@{?1^FVF3nuQl)#+K))T6 zQ^p{1oOWV{B7&$O0_6}V0Tm@@J~!i8+QhUIqy&s@Q>~#FS$7P>RwZ>kl(syXzX+c; zTd!-JmKfG2@9Fg4fX8}Z2geFyoAsyq#{(`UGPmY$zz-lNL4)Sfkp*PJk`Ww8V`#Vs$$~V{E1>yE_UX5*MYLD*ozH znfICh{_>UUGZUZ-+to>{U)hXW-ZC*3SlCyXb#@nm4Fs`JYSjz0t;aKvmAKURzG<;9 zH)yo@vZ4XdiC9kGt`)hf`bn)|rXltB1^mGLyjE#5W05)FE9R~-G0kDh>yRRGqlGdK zvVNLTHacH-E4W2SHy#A6$Nf7Hl4lt&5!j~0vAtioxSkmqaG>gRZ}2vG?-S)ZFq;Un zOuBgd0r`pqPe}t@h)DlY`l`6Fg<3tU$e}RBGo?sG=UC_DCupk5cW;ZDq7YL(NmDBA zR!EEwWDt?sjGZnRM3xs>``}`yUWov!KwLr3Jl$VQ#CpWa9zDccUvMy|1Xk_a=H0dWD{ zJw4ew{6(*1GRnK5g8@KjYgb?iAEaFL4fFa0*SB&M*j05w>j1)HYb#g$#J5KzDRB@Y zN?YVU;;-slVbS)nJi){&x;s=7q#3e-K?t105do{a58nWv%qFlr`R7`87UJIvrl;Y2 zy0rFcg19oHZ}-7%cDxb^HGl*UjpoCWV^k_4P=xYmBgu>>nchpJriemZvJeRkjzx}L z$Y5(~R>Tt@K=8U6l#c`E#v~1d3b(@89=X5syV`X%nK-rF zvEF`ft8U6W&(=m}7OV`c_usmN?X`KSuCzz!#JP9iM^b zIDRo2GSRce85EI5U4xaZ?x?Cs#BQnu2rv-MN=RKk1i=odp!n}lNp zHs>Wl*nh=YSz;y#KnK~;Z|>+`1sk8)yt=nl6=lUr>iG}}mr23gH<8K_O7GbSt@|Kc z9n3~(CB}JH1T;$Rrm9!`*`o!_WdsUDHgMWSqdNT1BIBPjy+-9P(n??C4OZ_B@+(H% zZf$+Fij>O$0Fno#`LzB-{-k;ITu$R<|u@B`{GldY{p4x&$ah7BdKAz?@KG8igr>?+McqF=R0K=S*x$|4b;=}O^URk z$od)K9i2Yr1oO)EVoMvoa<$jh8>j{67^2RhUfp@1$PM_Tf|$YD7TA@P@|Op2MSTnE zUwmU#x;*x!{KxoENHEY095CW~3x|6p7}5L|d#3P5AjAfGP9FEs_G61082Ju+nIgzV zDG*~-LjWXk>7id$s^JoQ)~^Z!(~SoOGZZ=T!BqUO)T4EI_ni9Giq~Cf(}oD3%DYuO zGz{hD0~c6)5^91%l)ghmD^J$3p4>jX#M6uQrAqy9i9Bcswmo`B@8UC#F+Vq~N&#BG zX(G6D@+B6PVFzipvCOR#29j|&)V&CH7PSQMVgvvXQ(fu>2xAy&@$rKmxE7AZT#6fY z-_iB&HgT%Vi=wnlqxxxbGwdV~r72d{Vq>mwLm=bAUr z{j*RM48}XwYS&XSqHM3#cH1bOu#{PQN4e=m??G4n@cgsYsXO-+ZH>s>DN^y`GEkLj zbViOig8<#6%%qR)`EN+>bQpOBYJBR9-w|_&!q-#5Q&*c$y0{dYF~g(Gh9C7iLuP?<)%ly{eU^PZl z8YxG+{(j)*Ev(!gV!L2UCsh@6OV=XMrJnOn0E{|a+!0AXVn_O^_i}9@j2Y<=5!2;vg)f82NGlniMH3lDsfP;;Hm;W1nLoK)T zo|4I!Z+{5p?%n9#@|j~>DL%<+jRlHMvXLH@`HTniOxU) zi-&7Yn03ANGqyE2hfFbJp>)sjiVx-GQc!~l{URrWEqfq$5O=;FK4(_K*y?C)ce{Nh zdRckw^_24xu=<4K({3!U5p7|Rhx)v(gKPDPl40`$wV`yCSGpJ;Xc7j6RWDRyzKkcJ zgASr|&iS@hIlHPNk@-zD*dBAfXQo?A-9cak~(U=+Bm;2@)+LUCdPB>St`G`dt`? zIRj7Wr?sm>wlW6{JMu)%Dqu|;S_?uCuY6h4feIHRUsVS-m)8|hnWnZ~&upd7@OHJk zecwz|U2n7?-V6z6iP3)0v5eZ&tmYN7e+HLXKhYAhwP9Ti}R8F>p;I4Z+C8ssm+ws@J$Dh#lb3Red0jfSgm2BS0y9# z$gWBY8jzqDu=-N~A8)E`&LrV6vEX$K1JbtJCmOT&9)f8KU?|a~R1WGf()7%h;@w6n zsYX92cLRXyC|TgZkAK|Q&QmE>yTg{Swz zp*4g>W{U#1>x!w?p`2=6F=yFC2Qu1bU$O*Wt+}@sTU^rQ8V-v6Rw{pS)CEO^sa}Uv^#@FnT4-7op71P8z;s zQuV{bqSdnYK0CJ5^ETFajlM9xEp=HPt@!C^BWhlbP)S=%;DDxC#;&Dr@Og%wcHx|w zX|WGq$6~C09{;=cbu}KgFZL6O<)?KYupoHy`P8?J;Ve?DfXXTAUOO zQt4$gx39lU6TFgPHjG≫Z7__U-T~Ya8RkGNN2ollXmXRL$<8R4}hv!GzWz4$)tb zpV_O|`Jl+|dB_tU0*hAq;Ukf;j|fd=w-#?9*~KbZW}BTJxCz6Ue>0tBn(4TCDsuz% zBieIR(H{irg+XjgpD8A3r>X&dy3S*ceM)o3!om?b{b88V?hwRQZXfqwGR(<%RA8&C z%|{XWeI7f*U-Wr7!OMHQ{i`E9xM;h&;!%F(r-26mFmA?ni#d48jKbOl3f|(vkrBC- zg-Qd-3*7q;Kd*s1R%ood&Yq0yZdSlW<0s7#te4A&y>c%G-iN%-^ebg4^&Gz^G)Ju? z6+9`p9c4Zw8*+@+NeZ8z-_N`~-S3;3w)#cVav40+B8pq4Ll3i3hW0r_H@_}(_f1fb zzgN%Vq#t=)_=a=M3~&cT_ZMJGRrDTx`54Fi;M)P!AZ|&s23=I`O0gr--8V;gsg)0O zfB%8t?d>7>`7$IA{n|S+A0~zpYO(K@`D0nx#EOi=);0N6u!A-#Asd@zfH#e4fw>9Y zsr=j53TFU%!k|#g@mlJ=U7UFu@@g4*Q2_p}5Dg*Vujb57uq<~h9hL*FwM2Q75g=p` zEQsC*{U{mSZ`$_$R817DOu>%o;{#Q6PvIYox7#l7RQe zmsK;R(~+L(v3=DQK8xV7j6Rm6eXg|qHJ9(&gLf9nW>=h{F=wbazfU;Vl;6mM@O$~_ zeaTl~qvQoIwFhD-Q;xf4oXolP1?w3r#}7JydpIMeR~zS41DPS>g)YJnJ(dLkC>Nq_ zqm#a5Qz$9R;YZ!=C2mqQHJt@^0Sq_>>-gt-WH2D3ogYWq1I!L^Y;C zdj9|blK#!ZDHLh~@V8;YfO3P{GB7aPsP|BT)%mzCCQq%krL6eiJ;p|Py|c#YN=JGQ zCCzWP6wLafNJ+3s%W~)`}z7d%raKHsyiz0g_;{ z>YLZoTfTj-0_tcyMr&hp?n9@ou0*jy^T{&IeZvf(&P)6oD&7?8v8x#6RU73C7_6^$ z@~Nw8A-vC#^Cw7_syh~~X=Tg@d}{r@RHC)J-N1NM;;ow5lcEuAJ!?~^Gf{^->g+zASNwGmo`N^6qCAiRWBqDH1roR$luQ|jDA$0@uP(6|CT z3O{Y>BPFQ51UB6HRaq!RjByyK(Hj0+?C%|Xj;}*7+`I#>8=i*L;)Xl&* z0(t6XZG19Iyhl*EK(IP-jJuLZcZ3h(t@&xc~-);F6rXT-JChPX(SzP6*FHFmJN30 zNKS3680-{cKKNhK-;@(P!WOHyR7`eTc<&XcnDEM2pWwXeWqeL|-1Y%2nfdHfPQ$8} zx`j$kX{=c8_+z)$tai0VPHik%<7zg$Rd!BmDB0}V^1U=BX)f9BGzUDSCbx2Pc=U13 zz-x}b?_09l*kOzK)@ur#)Zu`5yDAD&8#S#to?2?W^p^)@Y#8d9<|%l7kSmEkpxoAH zW4QAdRrA^R884;y=r)3f6|?`1J||9~2W(GHi~4ls*R-3I$bnDFs=&`=+=lKdu-w9X1Da!Z?$E(nS5!vS0$x~bcgui5OVqpd69Spe8oE6UCn-!n~h zIUHTdMJ1TPw7`1MRGE=`wtHF@zXCRgaviw@7#q=XaHsLaQ^Y(-(n}S#Si>>#j5k!W zULj^oa2MI1)6naGbu!=!g(Ec6uV~E2_V8SET%RZ^zch|z>WkTGv`cSdacy1moHGjF zJ#uBnI4`%~S%UhWYo~_DypHyPQ==)1kr%&M+-%r$3(joZ66Xi?;rcTYM2k|R2_TfLp94d&g}wo|5}DM(d79%C;_!PHZay!*qbnt8+UoOGF-$mpSfl@&Tzxp z#4#*cmpqgPkX&hn@ht9x2Vk`rItl>tl|mVm0~qHKiG4HYg@prH@}@zv&)?wB`vRmU z87-stWMOkNmw#YWE0=!+hr+Ss)yo&2BG+2)H%$>d7uY>zwi*sYw|@(;O>u`ZKVD~4 zfyCkEP-P^GKs4H_3H|1P1htgb+((sf6uV^w9Q&_LufMkHcN*198c|OMF!J$gZO#nR zjB{zljjhiJB644UZDAMO%yZv!;Ff~(iV3~e@7RRh8K9owljFT}W3PtswTVT&2h2TG zH@qsT{hJgNTYWC+Z%N@;XyYPxU5$y1Pd1o*U~k$d5ABGW>kqxr)7FV)ZgJOhMc3+P z^KCgzm1-_@H&8ovVW|P2Ke&WU6=+j4W($2)jZ#;cmZ)!EAdid5KuDQKRkz*pt5jdD zM>lIMafG>n?~h{1^9=ZeSG8muHEvc4iNnORFz`A|I}a}w90vmSAw91=pQyF)npZe7 ziH=g&lSQwv8ang3+GESJIhv@Hax5dqT?*BqM5$4i6#EQ*@Th7#n>@!N9mj+*qLym` zVB~ckl?(TBWmD#|f7;;0HZdq`Tw2s!S*E6e3>+cCKpru`i&ER4j?X;WJJ`js&}fB@ zbCB{lcTd~dd40!T5^k-rus$r#>+G;nK9ehLAX{3$f1+ZH@emB^3AJ>sdkAI1bD71Q zq;XS-hu5_xy=y*SKUY`J&o{eWA0MZeQ@ro%w7w|Do)Nzv$4|G{ zQ@ntf?3|w$zi4c88PC(g^hxhXtxu#uVlfu8G_N)%ot-lko5K0|flGmyD{*i)E|Lk^ zyR`zMm^bsE&~9a-(Xe|OBAU*3g)oM-Z-!D4HrHadForgQ~RAf zpi{xzGXIYMAgj_}doo4|KBLgTrO>av+S!YzeEdiM_8(cW=|fHPNUUtK1vQ4JzET8E zlE?)fRrIDuyh2WsPr?|tZwV(oSLKA5%O49t}r{V2Wn|3 z3{h2l=(c+2kaOdp6B@%zw)*7czqsod@nr!4x>kiI6N3%9;P5$KuAoE@%wWi#Jy!(< z-Z0WHxm2RdlBp8Xj<0l7!C4tQ??3MW5(hY*O7vBtOt^}zV=?a7$cA}LxxgfPP2!R7 z`J`Pjvy=pw-mwrC$MPI^=7UB)dRy&G+`I+G5oOZ~IhC%F^VCx$F{;uYMuIc5`uFfb zkzaDD$x^iPpuwp%%7oBZu@+!^*sC#XCBNMDLjSs>ykSAji}w&`qWZvz{Y%!&1QctFc#Y``$@=irDrf&M7zOw`= zAiN*^TZ+NO?v*Ict_Z7vB0Sn+<=_3L=2}U|q zVVK?elGaosfc$`3nc83CDwGR%xwNXBB(LlGVl-dUB6FDTNiuW}X zY39Md0y^}+SSBT@i*xmY4M%XOkDzIXl-A&pL6hbD_kB_qvO96YjWZVk&B>lE6~f;I zS!?Nbvq@K^d;AM^B%1XDBX#W_%Yi25F%#Z^rnVUxkKLr1EtVkG{fZW7kbj_#-X?2x zx>1=~USgZHcO7JUscoBUNJrxX{Z~;>aYt%&(5jTbsD>-HHbdknp0|#_X$@5cmu+r2 z$GUb0Rk50yxh`{H+ex#Fumv=bXS_aV(prh7f>fn_{DaaOTf?fJLUbQ|?>lGD3{<>5 z%SrRbTyWn4MimqK0Qk?CSc| zgiycK5nYN?k{W z=!$CHG0VIG3=SByABRkl^XX122J^Nd(qJuGpN?!7+`6PZvpGmIHDpL zpm=r*-?5+u>dMG}0b79P%?xk=%vgl=~#|zZA^>3htIUPo#ez^O#Qg)pPXu zp?TkzGLXjUsRD4erL_gW_g@>i)yB3PcLR>_FM4-hAwD2%eQD#-SAGJkce+Ha$noPMzz{Gw3T$9m8~sK_YLi(y#t+*7w0(=}W(CE2%AroeFER2~^4p;0OP z7a;($>Ento93}^I@`MUGdt2{eluz1D*5F{UW_mV(uLFB<7hP~&IdYnfgOUXSj3%PL z)IZ~U8Aikvnun6!AleP{Yfr%C?a}Q`WXD(X{+-)K>OQ~C3TKBIt^5v;OvX#k zV&lg(UUSk_X2k1vjPKiE|MHMy8TD8`zDE5+qmJ1uzg!=#r6pM3UxVTNfZe+Gb-=Mn z$f;rIa1Wekpy-eqVC>`(f}Ivz%#MTEbjlz>6E1qr5k@&R$e;mam^XS@2Xa^mQtl0U zP<8{&a58*4j@~)>;eug|U(f^LJ1I+N>bhFalYUf&Z%eWllrwVRfs3dzMq?cVe;XRYL6mgjqFj`67} zBZIIYQmjNEtbc1t($mh};4T29q_=a9yp6`vjy;X-@&bRr3A4t?MZ4e@FbqqVM&2Sb@YIW}80}3IjKHkW5 zYMkJ+j_zWwq(Opl{CFU#<>s&&=Tp>)G7g_ai>bwKlD021VjSt=g7CK&Ga_GTjfF-r za~AqCJnM1h&@v`GBMEW7Eq*qyc?tn~-}pNpv?_&%q_16AD*L(Jn^v&(1*Sy!uxN0J zj#h^b+|ZjHGK~BZr6url>0K1dZF{L z&fdFWMTYPK4o+I9-7LuxswBWW8)5Z0#%?qKqw&4MD7qaVI%I-Fv8*y#(LhebVRtgs3laCpM7mxRj zUlFa|w(g+ZmLwXh2=ZyfIi%W8Jklt+PVh$#F}G$;ANH+R;2no%p=2NO6VFFtB`)+b zR^|d|Wjh=lU9Y#3u|Mdt{TUsWHi~3oFmShz`)P0s4Z5%;+wS%d>oq<{aFMxB2?rLE zo_4j_mLMHcTA~~WH4?wvgbn5%w>7~}F3C!D+faiJk@P+_rtkT6lMa!GSQ1kLX7sjp zY%AVg4#yv9va7XREE^z)uV7mAC@~_m#g(R;=yfA+ zSjrb@Js6r|u{51V!MNWymj|!Uuk&Yf!_?a>?op4!m-pxO#=jeWXHr|N{B9`>`!Q1< ztyd)H;>y!5JyS&xu&T^yoUDY_?8Kersc}4axEc8k_7c0=T3T^g5}6EBq#H#YrSm*6 z>5zqV7mTH#iyRW>t_xpId242P;ZMm$X$;@N8E3pM2S_2A z$%aWAkK`ZxV?qOEhQvjzAzleO0-1tpEo@VGCU5RzkfTW%SefDues zFvM5y?Q*9`jZj{*Q(EW=G@pQRc&_iw!B@XaI*nB`sXk^i$g5pQ>+iD@6BOhQy#4pN zsi_DwHw^)BnM5DNlCW$KpI-RA69AOPjN2#v*;=PF25*Sloe8J1ne@aqL9I?}zXPb2 zwv=9A^XHW-S)jL}M3YmDrS^!CajL-igEg!My)IQfbgE@H3JFhLXsI& zX&hw9VsRv*y?-9kE+vdBiFByauCPmX?$;b8$(Tr)1k%m=3-v~iplQjwPtD>Is>Mvx z{&nt}#F?hj|3t5;#H=r_lfMUx>$g%FFSjSSZ^tu~=ZD~$TezcyEm<2e^F6LUbNUr? zGf=wm<+xm1cIc?fpj}5?77J43S z0xyZ$cHld$4@}B1FK^d&IH;~eL5qQ09kY;>4o*s3PYSWLH4V2Ol?hDxuD0T8HFlJb zj|z}oz}FKk`9Jq4FtLICL8TMrLC0U5hd7gN^|2`C#63hOpSnu9v0R{mTOZ-&T!A{{ z+72NL+(V^rLr(v=db-;Q~|Sbt&*Dgeefhot(_NHW>< zI8o+&tzXcbK$-`#=7ozW7gJx&q8SRYqZWE+A6`yXksxK#@Mop&QOgo`Ot!EuL~wp6 z`8K5s1KFE^M=fI+aPqgUq9Fbw$u{{s3%4cG&&#nLUt7=na~Z>2WkgQ1m=Jx~wkvY) z-4G?UuqjWF(9~NRCyqJ%(8L;fQsCC}b1mj3I>Dfj6T%vKiqqo;H@Iq^pSrzIu%~u| zY($$_Fj_`FzbxGR$AoDiFgr<+-^X|7s`3HlIPYJSSjruM?WO`>xCcqa-_KihBRTi$ zsvA-s-*#OnncahomQ2hrC$ir0B6eT21@W$ysAfjj=OL2H9JP|TTW|E_qAOcs1VCU! zYQ+hMU0%Cwu42G#e>bCkM#tB#c@$bh>y_S$8#=>*fc|UV77N_ty_xIYO!MBv@3=6R z7V>C3^xk0O-Y|AJgc||A75ldde)m%FUK^Kr2Gz}P(7kHU3*BDs+l(-16te1fcU05) zQHh^xiRDXK@`hRrOfiUc<#nDxR7t(KPh}o%l@ie!RT=<#v{B~so0g-W_-MTkH2-2B z2JQZv4~qX32y?m-dn0M)-nE8wNJJOq+(;(_;8*}23BWnhG0F0^%uw+Fhz0NFa_17n zqG&u&X$@`?h7#r`T~2t73xpV;bH$lQBGft;kCMdg>D_AFM8j+q;ybR+6)EqSjMN=%8_JcfvhD=#RvPQiaVob(-mLzW6 zCMt@Yky-kXr#xE-IdceETf~~+Dx1RDaljMA-w^t>;YU{&TpDrKs^co@h)5;f5@OA= zLTsRH5wtgLi;0e?Q5}ovDdJO+R=DMMWa>evYC#0Nep9-05@b_4sAaqeOUpx9Y`_m_ ztq#2GEYqMu3pEg6W}E683mrZDW=e>G%5F{>I3JBs)q$6^6fBV=nBP0RNWhGI>61XO z@wy$X{wZ7-Oc@Ov0;>Jczl@tz*TL^ILzU6bZBj(A>41xw^D{)1oaI{Oe#}4V%)=nm z%DpCqrc|mdmh!n-*jI5z*;v=vHOb*mF0l%>}+o%-%+ z9uwh)#ZwFX&TNu7s=Rbj=n0mdoxhUA1$E&&5sitdbxfEQ7#XR>ag%<}(P4{u(kTVh z+@D-M-gHJTlR2R)huB_rGAOCmX*5v|1rq;uU#=H!f0n;BOxHYp+Bk}woV4L-zEjX% zqm-_a;$Fr1gTb_>%lKTn>4>`H!N;m#b7TZ79r`7~R%OsOjlGyWunrNxX1!t`OjFcG zA|Pwg`)VxJc5h)+7?j-Ogg36RpJoHs05kZ}c zlr_g}keQN`CPFhgrs>+Av604g{KJ?a!<0p4kq16QeRVt2$MAG(0ymy(WeNtDH>A5+DIF z)uMxOK5$2Rf(nynr%ipZ$znFSoKU8sdU@&8CJGwNzQae`&|iMBeiClgkPir|6?T7= z=BBKyCIm$A{KDTyL(lsu#XDGI2;s(K%`%DC&7#d1cD0qXTcj07zQ!(hP3~UhO|@?i zR&;}GcdZ6e@TPwHb;>aPg`~%Yf&nEAX^A81wS#E~dk##WnU|!?w>f_2T?_b_g-?yI z8-7fng@!acA`tg8e369v`1rkfah#+|lI*Mm$bZaD;x0&ja40@N>;x-?=S$x3g)jhq zdWoe7z^u_+6m;uW^l9X->k*j>wAyDQ9Psnb?QO=kdns!vw|~@o@e@@y4Nc#ypN^iY z>`#V)$|bPY2t|Q!1LS@qEwwEKCUcmA0eeh~*Xs00^%$4MEbOh>6Gm6hZ~pay&oadE zvF~q=?2ob4T%tskEFt>9FNtyqTxkhKCJ86{;rU{z?C)@?LCRz1G zh6M3ZlCrTj^pJoY5kZE$X>uML{$0|9@uS2E<741_c9Yi(Bg8@0?l(?1pKYFx<51H( zCV;R72fiJQc`C1irfQI{c`Br{cvE@BZ6Cu+HbcG>+w;xRMblhay7gF%iRMiE zEA{Ol*{V;(^6E z3!bji)nWo!NU?;^VfcEQpde!9X8t?!9IA+5$=@m&$kJzDxnn<;PW;JsszYk!Qr)p-YN6(T-PuJplo2xvZdoDrj z4Z|a7b)J%$lz?a)gal)O7JU6E+#xsZcb($|C~-5%oMqBeOU?Tx4R3xSiPRLjc0$wj z^WpcG9G#V3ox>%|FJ_b70si?B_L_@`p>>CottF|A62%mLc z%N%vhf&8l-0oH6VEsVeSqSpnlIa3DxBYFyB89J7j$V7)1x(DIRg}^Dx^{pGjXi1fH z5LiD~12X;LvKm$NiK3OQ&i51brS0D0yVgmxc~?A$STO6)EPXy1R&MqZE~SOasmwLNHooO?d(Xicah6{bc3c&3ziwsk1lhwu`X31ro5Zjy`Z zMXW<-E1$Ic^rqf!L4?*hOCj#fUu~o`wdz^0Q^C-`;UmH)**F`ApO{1FOV*jq`C)!_ zbYEte%q<(DHajzI56no&%f>=b(a9H$%&p_2&!fX3TBczm{ntFVX7dm9!-u-=*n5u~ zPIQE3yfTOd9^>`4*zHT0nM*gd0Q*CmIKhjSQus(tts_MMi(=7ToJgfdhYbRFMR_5n zd5kzl>FdZuvF&{Ex9!^kGnMtRGQn6WYvAofSwf=R!}dg3jP2A!*~%u`hhd*6{gm}g zl$9c#*EM)*G>8aFF6yt47y}|+`jDxMm^djO@A8Cj2K|OB6S&5i4xG2vBd0lbCWUL^ z#BP$rbQ5)xL1SBFNx&i8p@n9TKIP(?jW&}VSH3fT*;|gVA-Njr?aIP#N~vynZc$-9 zG94NT)OrYLl&;VnugFSVScEk;yvT-f53JvGlXk4s2tHkRR^r5h_R3Iq>L3Cs@6s{S z+r}3(1-x1N;tY5oiydBevt3s8P|`=D7L*~_GdTp*c8x>Bc#ZSvpEzsvus~NMvUEb{yIkE zGMrPn5#g9=9x_N*Me-@Sz<+0>JhiTKJ#r`_Gj*iJ{a4I8fg3hM~v^xh=ab|_>-+M{X|CvpT? zx`hQyz5xRpq8%};ZJ>dTQYB2pruhRK4LZdooYGNku}O!NRB=f~fh$}cmhAggYcK(r z#OjbI(fc*1n2OoT7|_z(WP|uoY%Ivl%WG6@veP~pOyCUeGnG6R!t~n0t)DJGsULy* z7!X_Z(}DAtnUwv0Zu3P&G&VyCK9WdDQAdn4m|;twf?^zFJp24KKd^bQqd2!H{~Y>j$1^HZJa zTT_$R86bL87%j=LInyf1Sx1i;hBR$_RDL_TC_zTjP(~cO-pY~~E5i|)!y90HeG8gJ z&fn^|pW3|@r%0%^!O92CIOyfY=Zb2>sKW42bF3K~gm{|?QQ+%bv{8_^(ixf)O2Q@m zgn^?Dp8j3boe>JN$avz`ErcDqh_tp25)_oqWh|3Jw-z6@+Oyxi zp?pE3(=|1%{ME+@c;Z;yRU+4AK5@0NMwQ=H5UNjHS1N22OuJE#prIBv_Pa?U z{}^GqteFU12R8a7=z8=JsiEhZO&--PpXRH*>);&jl^I{uE#EoG3~az=*M(Av8tfXU zJF><%j+WtCGmt!PJd+@tG8Y-b_s8G;2AX5d5 zgNaNyB!$S)QgF7UVbF@R7`((($*|(iB11!vSb{7KcEArrm3tgII(b2WE&(;zYU~4T z|C@-*Nud^+j^mKuzvt$Bj@bc6A24XjgrLnPY9=JjEX+J4&8QpV=HW3ja}(EKEs2{{ z+k^?5*o@6~GV&uFBbIYGQ zWHZTIY&d*JSrj%Kr<0<<1|R}>NYrFS&0`@TFuD!xsOQ5rQrBti)&d1rHd<51LQYTyh5PHwg~ zMPg>= z1_C9_Elk}TVrS;Y4Nsx;=F@1PsU-Ufzpob`?s|AV71}!^+F|}qitQlZ1x;N-4V7)f zmIPJpxSHPZN0n7X3Xu0l#2Ebr23v?q+1zi@9SOgz7*+6zNyk1iD~Nqyz7csIou;i6 zwsg%D-5RpsT`^+ZmcZjP30Mk3D50zf0MfwJxnhrIxU{(}!J#BD`QwAvoN+8GkF*b# z`9y24nYbMq7Z*INd#QqNTKhO_%_iGN@Vqn)ky74ySCFpVSunVjD0p$Q5;glvo1u&B zG;K--n+CmFn-8LqaWb}(%2@a26BnqWl}I|?7N_e(;yV_3eAee3TxZMR#Ms}xa_C>fOVh!moQ0% zJ{1|O=5q6QmfEq$9s({`B(BsS<2#K!3{v=4oaQh~tL7cd_T`w9 zl7Ekl@Pm(QKJSl0Yv6!1gskag6mQ^O9&2;iu;yHCOY>m7Lp?hgTf}=?qw|ybHpDAn zMciqpzAf~K213D;Yf_Oe5GU&NIyc*HETW79ZsyMrUd18P@Ue$VIv6}+Lj&{y3Xy%( z4sZ|+KRPScai9xA&pD&z{*v%D7|Y*-H%3xL`YU;VJye%^XtiS;PCyLaP_)FlQPi(< zw_+?gOvn6;qAL$uYhSWnHyUfj^uU$F0I>C6M60q7rAmto6i$Z(2jfJ1%bd(%lDc5V zVXw~V)DPIy&c7|}=ozy)=8Eu)#6rFXT(vE!+^52GjD!sst0@*k;KHf}wv19Yw8Id* zaw(kpcQn*8>6zWY*t6Dps0bG`Xt2kHf#A(>R!Iz%<$Ntra>!|m*p$py8qd5>tQcm_ zW8kZ~mzwJk4cefQdq-UtjxUnuPQC@mB5lSKvggQBZEf-3QuaX4!}rJH9wb$8KHrIo zW~WvB{y?*Q*6#~sOL8J=07oQ87S)F{kVqi%{axmM*F;ZnmR?}w(VA@IU~!kjhzTt6 zP>a&q9bBMK*ld2VKWf_ZwEBorRbJq1ep8lZ6Q~vy0*b4swO0FAwKK9BabEBJzw=6T zxO!~omKRub_{F6rEv_+X2=%dPaj)#!++jVO+T7gRmKP0n|DVSR#FZAK2)vSIe-0IB zC}~(#d8&rqbe7-Ok7KI(0=sxKtzvp`l>7oKTUX0!OV{G|3TW#7%caz7nLU26%Xn{u zmo(s7_iaTJL&2>pe}0XhEL1Tf=)?(bC1jbUSVb*`q&nBsDLiom-PnWo+>2Ja5Cr=P?n?va}cFaWWTFida`Gz+a^BI?lbSe z%SE@{w(l~yeU~*Afq%`%S1z(|W#H5n!pp~=Zi@ZGv(n=M-CTZ6A&E^OOoOv$OtU4i zO0>+=XzYejAQnm;*%1#|bqdRbj7`H2uF>PwIn!XN4B#~!L95tiLd;rLGfdDjsia+Z z*RC8>dwi_jh{D5R&aJ6|?+D|ydG2ktS-II|9zQc)JWK-SRF#m3tJJs9=l=4AGFgt@ z4QD2`nGv_xgO8c$_f&lm_|!_AyM4$?bt9=RuF*`Ow}e+kfrI(P&Wm?2807bcdG=g6 z_UvZ%glF^!y+R`(+M9vo(EFE$GS`w-1z}Y=F+8J6=n2S_V3xVrCVf#l%k?xxU6IHR zG$$-8XPTV$!cFc8X(T!*E4i7K1U)dvMSBXAl^~q3eI}9<2mSM@^aut2^hyAu6ORn` zWO=(w$o+n@=-WlOm+x)Rw-b+Z-Sj4yj(FIKKXD89NBv;s6F9}oQE=K2?Ts^#w%>}D zgE7EN)HOcn+#R{$5o?f|qzhKUBRE*uy+8}##yE8ZUUbb0b>al7=#JyVY`v$4+j{rU zZl&+7UFnP8dQalUTNAhS?#gbZFMjJy>FWn_p)VZL5-zlQ9AV~fO+kzvgsD%0Va#5I zv60~ZsG{P(dtgjL?u|oG2yI(J(}xA1XhA4MmpMe(s`mu_H{ele=Gk@z%|nBbv??;X z>B9D)aZ-ks+EK;=XDOL@6TakF2q&;s7M$oT)f*B@>m_$~6SL&Trb*3B)9BKq#wIBR z+$UeQ7N$K73EU^&*pI$39|I!0nYY&D!Eql0)0wx`g)tukI=h)Sw$m@Kfg=;MoRGUJ+?rZu+4+jpWD|5uGkm0R1^ z+$?E~O_kHu|8h)~vwk4yuYDvRF6Uq4#J0(aVUr)jp)iR}ZXBD;Fsp5h)mj^)wJ}L+ zWt!e92pp%iDZGpMA9`gs*Q(yqJTk3uT3Y+bsQewrp@_ZV$jWx8ae72H>}(vH^!ma= z+SNQNDgGzExci?Z^BO01>z&^JevIFG4-%f&dQW5zyY;U4)pw<@ye)R{i-1u7NxgMiZ0nw52vg{Od*FhXclXgjk zUJo5b4LkZD&H5&+oo)2a1Y$F^E>|FaVTrXESbTvcI>=*yP!Mb3jQ+-*Ed6v5S!OF5&Wi>wM6O=GdDmf|yvGyfR>9Vh4I z0idAbGomp~E8Xh!b8?PPg5HoD-`YxVOuh4eLs^&a^JE0Z*zn?+!Mri zoy6<3bT(ki?V$*5M2u8d2O~~B2$%Cd7kbweHh=-fp=EttgGVHJtCv=jP#bwp zYjoZM#8e=O*Fn7gDzE=`WX@px;gAo0Ywth93>;`VcnfKN>)l3)LfyM%7q{DjVv_L~ zw`(IMq)q!z*|xXw2Ob+u4Kah?Ov}Obi(T-!?-Ec~+=scscaYm-1{+gDm^CG6f7_Y5 zGra%6$B2(SyJ#U6Mcm%HX-}m97jf&lj6Q8uFxk|0l77o)QQj)vh`sC)Vuj-zKG*Ko?q zGtAG;2aic-tjSiHB%xox{S-v)G?Fs?Vrb2jM1{c3JWI zDp35sf0Xxsb8Q46TgJhx!xeYH;C9DP1LNaX=_%t9XHuhdz+xCVf)~|6V9E>oQX_>i z&a6+K=ZMg1lGYT9UIs0h`ZvLQ}G?({*16RV3y>W9>B76uAC{q`nT zi$CWF6Swt<3$CV`l4qDxTD;l46K1-S&K*_nU#eD*vJ^Lu;qCN&4S|x%+u^@!Spl=7 zrF&ME{DHk$Hi6rWQR!DHs(D--aMp`mgEzKd`_-{o0UbA-Re^}xWU2aVD$e3zI96f$ zD818gO_SGP(=4$b#b7URb?hdR%9m)$G|gK6uf-#_laX|#i=`0aHAyV#0-j?Q2si#t z4ic?73+=j|Oa6&Vxc)PbApGwj_-DUe%6n_HcrHI}w{tAV_ee?tM<~qGS9W8P>QG5nebvmL($n4oE0-&j6=GVj~e!GU3 z1!s3h37e+EO^z-9A>>L8i$x-*b%q?yxlb>1i8WeN4RJdMBn!0i>04}blxzIy2_6g-7K5FrnEs^i*h_1^ELj3TA&+8MiClk!)~GVPJYMeqJ#ygb z49LclMTri(qWr{GngRoU5ZLPB_%Ob-+KtE~MJN&Tohq5OrqBw@z+ zj)E1#Z%PPtE>y1m$kBZ`)8~qxl;vj2(6fod<1SlgGjm$^(VRo=MqzrFiUU)+UUzhr zg}KGea;7yo=sDhJ>fYuNk!WLMOR~=HY-f7BC0=KDyw!HR1v6?2WsE0_sy7~NS>$b1 z5=kP5XLTrvy1mY}G?GZ}F3?3M2Z!2yRdiJs;;@`PNSm~{5hv7wZF*Oh*tPGgo7~ea z;WgT%T%R4&i;Nb zz@Sy?;K03 zfpv=rF2N-~X4Up?ZJ3^C8HKRCBQ--rni!8aNjua|0}+f3&SWdlOh)xY0}fyLb$$DC z0Y8Lx0@FfJOdy6y!6=sH;_Eq;6?iSaiBgAn*$^Ui-N|#k9&S7cH`rzYQ>mDS+;!GlQ(ToGM7vHWD1wA_&8#+m3$?K+-~?h z1V_)|GK#(c#u-j)_hQ+o&{aJx_vz9J+H{J}PQ_kOz5E}?kEe^x&Fbmt{C?jT1A+Yh z9(EB7zAiQ9E}E8>MLi=|4Xj;sagWeNy+NB!v?l^WuqG?!P!Bv{)_pus?3UclPwrmt zE3ShV=8Qo{@=fA_l;*mmrY>}eg+|nZv*r+U4x>)*huWqDMXjPS#NwDV@pIV1!vh)& zXr5Hi^q4!J5{n4k0fLTpne_+mw=fb2M#o+eP6$Noj%f{xI|DsWpPh-3+V;dq+)Pep zW+xI<4rZsXC$3dqv!e4Bqt!}K{&?-1oJnYFvnv9Fw&giDV8^pi?Xb#C>fYyrWdlS+ ze;@9TaRq-qZG2y)AK$Zo_}=~uz5Ob84k!NgaebVZ%65=?V7Zjf!!UcZ>**t1_F5-L zWis~Tc_C#;g~1!8EXx#@@R`%gMH&-tgN3`ylV{aPF$EbtW+)APH;&~mD~b;vVW+`N zbD7@D@#s;NWdJ5HB0S3X0DD)VwbVd#RS4Bz>*E9II=U9_g?(j#LLW@*8Sg|$)!eN- zNRs%u@Wk=c(FxEGrY4QN7$3*Jm~ZyKEKRd+_(R2sNc^=!C4pGU0VnvHU3434%n4>B zpfW+B7+|=Xdp6#DcvP(f!<(0}mK$s^n7SU9BxR0t+EIpz&n?*7Hq;Z6T#d0Q&HmBi zQ(Ve(@bD7%uW_a<3APZ3=!<=ZI!#9BMP3)gBXjb=m>rzb^BQ9)0DTLqCh;VZNjpZU z><{>5AkrMOt@Lf8YbYAhBk(w}iP6Y{;>n*R>lkXF`q=+}Vr1_ENJy6Z`w|ne)qwO| zu+)HX?NotdQd~-M^mJI2=MtUcr~~39!3RqG5>WnBftd%af zGJw-uK6l$%5wBlfymEc!#_yGvs8^{kT;JO+{OIo=9zeO$kKl3%S)zA&BY<)ouz$Au z0h1Iwi`QdYBTv|AJQ)XExQPw5EBFKf+URu-#_sTkK_2})rcH&Pax9`5P?2|88NvR6 zUAHt+WRt34V=eF&K`&v=IEplPeE#HvC(8}wF?Ry#*B?G*fv$3(%*>&7{gJ7XB&S6N ztEe9D6Ug`bjK5F`W+o*m%Hq#F_^vc{hiA&H#oOEiTd(_4aEarnm`&$2^kA4{=%v@K z1A^1n4jg9Nq1#Z=J||^ z)s8GJG<%lj;cYrDswOAvP%D@w=Kdk9T#IPU1d7LsB~~*1G<9L<;yMcu&M>+*^nRHB zRBt`A`ztsGF>$lAbR%vaZhpPXUW!`u-W|Lg&RLt^ny7I2iez)F@UGlmS~3i z-)oO{ddU4F*r&yHmlAL_#spQ%1Z2sPxQWSjkr3iUTW3Z;gfCK4ySQpeLfxyds zvP7l&USW++wXvhTy})Bfei4Rt#2cSbs>d0hd^VGuU{lRYdg5$Rs{4H#n>pZQ{Ua^d zBwmyP+J~3oXr2~jAxc=z)-`9aGJ9>F2bMBzlObFt!FikWeYUVQSUR8Q{A7vyiJ`N5 zwy241g53K^K%?tcn0stc?5T?(H>bxn@E~JOWT_)dt*)n67zDkyrV?w^)bHD*&Yna( zmABGrFmaB&_ydTDu_20|42`(iOE5YVj>2ws39=<}A+q3&oBfhwOQj=-*JLv*n2&-o z-hb{keL`9MlVBOa#3JN=4yuy8Pz~L2R`pg$7@Vti%Mql}+S6gPvWYK!P0AkR)uIH90J-R&6hCdvo zu~RW>egU{!ac1AIYBNZJT}egWtwofj2qpiOi_|x(-{FLqonWm?t7_)9nm54m4 zO3yrM$XfKy5F7V?2NxeBtVm~(R`?WlpmzN6+&o{d5a@1pT{p|~Q@emMH3LnD!c+_` zW#}~oC-lV2#5xRR?H)C;btuyqaAoMpIsCbI2GMP^9WDpez;*kPQzI&LOBn!dw5clz z%0*iG#!ILC#sBw{VsdL^TvD$W^Jr_1-B4H~P~7Sm5CuyGFPvc&O{!}(jQYN3p*AC&`V~>-hYxYP z$<(RyXeqUlWXvlZ`o(^66B>b=IjF&^?X^b3vd$^~in8G@*P7T*pBZWwcd<8~qM^Q->P%I_oICx{PtDnlI84e)sEBM9V*+S)BMIP4L{vpuU17~tEpNZEhp{t^ z9j){bU_%I?I1qf+IPH^E@AnE&E;Q*j51G(3joESDns{PQnm3Q}IcdtyoV}MvS>_jZ zAHq2kGApKh$8^>Ue0O^S{mVg~L>4qA+s0d(&WdVX^mm4sx*nI}-))-IBrs_Yl?#cV zCB0S?anZumLpL=F7iG^a?SWMa%HK`I5yeefmW*2mGrqJGP6%#e(QsWJ;jz@&eMk`212;07+`rPp=~JzaBnosq7Dx!J!qks-}%3AsHwE{ zMF{w*gAE8_Ki3s)Vy|Ji#?(cfIz|1k)~Qxrzd`p@*+~@suYO)jY00QM^r4w309jqJ zA+}^zB86}Lvg1_j?9m^t0WTK!Y0`@D8!9XaJ0Y_i#?`vlkS_zOe>;Il?(?Y-#EH;+ zFUB0vyzGCw@fC+u+q0^0-GV|~P*mh70D62vWj-j8tfgAKg|7&*(w5vG$+XNQ6>^pz zX!GQ*MQJr|={kwwe9r?jhICZHjjJn>VWHRS6k8txXi7cQ4D*hrU0d3H);De6RgxxY)q8?>#f3rhw0|Ptq7@7^|AdBU9f;Vj0fc>2T$aDf>$3Fo zd((3M`TRVtZdX6m&l&XYMLr<8ih-sr*{Bl+y?A$3SqjrhS8;()% zO=CGf^(7JG8@FSI#!#a=qjnHfxc{r)}F=E{UGp_=l+6vo25r@-D_I)W+%K< z>C8;)TakJF@EIiVR6t^{p+qZUjYe}p z?svp-6;m#&MAj)k?ZR6Z2r_!e&`mcWcO64{IU4Nu{krV`IodC``Q`YtV+r~|0F^<$ zama$)+|CI4USZ8o@CqsCR#+xe`1~RE!G!tnSsw6tIy~VJy4BVp^E$h<$)(+tGL&zS zsnS{Ch3%D#)|sqpbG1EUYArd^SX?~;4&c8svuK1LA$Q@}q-z7xy$ikLR{8KM-azgE zopYcIL$h1!HA0-m50XC}EPuC@9EA!9Bl*2lN{Gg-r3E@I*}*(#KlIZ$A1@rRw$Qs5 zK7S(WV~>42+;C@-)1p4E`kr_slZn;kP|}v{G8{hvde{DdHmH#^da_f66b)tVF=?ub z(!R!NOwmDCwm{a|ZLU>xdD*M-Jz zGt8<5fD1APPDh{~WSU3Mk>X|F!Bza(8=!W&@}hvr#oVt-QMO|c-G6QV(xp^n-neQC z+sF@vmMisG{>A%wGw{#F^L6@b`7i6oJE~7!!T|5zCI9!!`T70*xWk^0uiL{Vh2Qg4 zyjCJ$zuW-P-?ZKj=KA>jJX(&hvP@mXhIbj2h@#_#Vc?Cs0JU9cdnf%3p;1G#npDgi z=!M92MDF~Eh~ym5OLG!YwR$}*wsYRq3;49ISHEFmqwcP9_z8eHE6y`tS#GhNt0?~< zxw5hbb_v|(rnVyBmX%B}lt_FiP-=cr{P3Q9kXacH`4n{nY>g6)rp z(@D}l5*K!jqHzz93@AIQ;n>oIlbS;AvT3qdCohu{@rp>>aa-aSN)mrnA0v^}YblS~u9_H~y)++Qt%xAbR=lH% z6{C7B{2YvLMhqOigU;$zZ`{?LBs}LCgH|*9E8Amm^7uC*X9%*v-z^QxdKUIu-bMYo zOWW7&FNc0o zw{MN55oSk^NlXY`6!&Qqn*5Cg<%IfH$G^EyQea_j?jap3nD9l_o%@jmjUK;fQt$92 z3|e^+YS#bGOau&jR{e|L$Oi{7#gcR*LL_Q&eilqZj!{S%l0uFbjJ~k@rUQg?$vs?4 z<0f_7r;nMq^83+0LCoe~J$xDTuLJJ6JQEdXQ^>RE1ts=%2@609_m~bJc>wCAiEzkG zgb1yW@rGjmwP5xY6EY%~a0j2)To!cfT7lMu!9 z-XjqGvddXDh0ziKY!m>A)6J z&74L_GgJy*ym0I*dRdbx76sYD0&oldnF>mzq+oG2z;nT`<$!IAEuKHVfX%w@;shX=Jt8@_WUxGD*paM!OiZmNPJ&kWgW=4JZhg)4)F(bhQ@+zFXO1mg%|#uu zyNkEu6k93GJ18J1GVH#>7O|3wH{eVM8_=M<09e88cQuOIC8k#8n(@7E*l5G<@0XWD zpZr_dV18C~{WWh_|42~FR7X;(9@!_PV9fcskK>lHbb&@9v&2gLBK;jwC z7Iq-3%U`FdW^>(JaUA%Er2;lHP3CUx|OMec$u~G6CHqr;o za@w#w4x!&w0q?cU3^dP(wKJY+H=I2=_MZ*v&uzSm> zcVASXkS2PYd;M;o<@|0B9$2i)FiK@yQ%QE#JFy37deBC~MoIx(z3IEX zs#8$a!NFOGSlBU>60w6xs8`9lO)Jr(MKKU9U>P3xC`>dww%s>!fP0O6nir7v2Q^^? z?ft)Ku{MSjjW}C2S8rhUT~iFA&dm$H(H#$}RmWAF-P{9%GlspsZmXh9hIni0bGa{Z zt7ljapWvszF1n_l_i4-Af~n3~7qX<74^qFVY6~XlUnqngS_TSgn(>Zu9>Jbl#KJCc z2CM0(WNJ(58GiSj&j)!dR&7$*K+UqnJy}U@%}f<=vQCJ=#^z$LrDVR4(74VUb(>`9 zxF=u#G()+s8>2O%WwvQE*C0y8FMdBC_Nb;tX0?L+3sys>%T;GFk0ykXZt-7GCX`K7%@h8u|;d8xf!N`R<| zlYqmS20|^G5`N<|=JqAsjR@s&&@f>*#3XYL(_3}M3wjc%DzO}n+rmvjRPv^OCFLYW zTD;g49}~CM2ssxMv(~uC&8|KEnNLt_lAVlSYf=}x*0_%zv(^Y;*B%$Y+OWjOtu^Vj zYl&-ZU*uj}7QfQe`7en5ZP2$ID-JBH5(MlxMR-D##aW3puO$8y42K~Drt^~{n)Sf02@s% zUbJk6?XbOcFMwK0MYN)#oel9ZLuk!%3f0963An4IT~WO~f_RKuhB}FD@@?6QE4s;R zzPWClkLq;!tQU8CoQlqrAWppB&)?~^P$`RxcB)w?aR5QlX9DS7Z+`^|p{XvIMOH&k zt|dUrBr&AYtip?75)`{`L4-JXM7BFZPbEV}hkF+KkO1+js>({}F+@Fe-cX?*xW%>2 zbaB?Za7{Y&1GWB6g@nv=t5^+xl-^`;%9t3IhpH zg@$caSd$-jJ=rx&^%c*7AT7?&)9Yim5GV5hTsI-A?YSNTvS)bE?NO#Kp{YGQc_-*h z&9GdJ<^h6^L|#oa8C;CZSL8Mq}Tgk=VVB{~E#Xm4iETj~7wKYEA3&k;uCbA!D1 zbF$eYvsr}rB$F(4TypwNA$9a$cHScHpEqH#h@)p+)vrlgtb1X&5?`guG1@~25JWi^ z^-NEo=M0l`qpq$@dkd7Y?Sj~zeK|By@O86Eh&FS$P(yxA;u$ejF0)UgKwJZpME;b>PQ(3e@7LTL)$5U5H5dWzeI{8iMnQOhLuYz6lvRx{5^CVP0pKL zbn)iRPSy1PrRzrN5ENgon0oj5f3r z*N@HsoVU2LGSfYxju}x!0}#~{oq9oH_nH<+F+4Vv2o;O8$rf9R0iFvuLW9vy+KcYLZLByI#SW8F}zgtt!=^X@!|@L!ZG=zHzXfj`aAc{R41#@IWA`h2`Up01V|40`SN zo-|msKb;=13j-X0;%P7FNK(+Se=Uq|9K+4>@!PpKvdhyUgSp?jtE9rjo)!azDIrqX z*Zwj-wz1CDCUip9*-dgs@$~q-U83YDx2H`Acw%P)LR(_G{Lj8F_ zitAk;qCuXKW5D_6CYd$>u|oRYsI#x8 zP=7|tfgZD0L0IOaG$Zr3EU97EP2X*)e?%+`Ci#vLAfG<^9_||l*dk|pP^EiYS85Ec zim|C6^i{j_eR+F=sd>z+;E~|(b_KA2q&J|a+rs+2HJ)qB?l?0!5}=-r6X2d}s?Lb= z0siM0Q*#21jQZ&A3}PJ>YdYZb8M-LTH3MS!Dj6xAF{=~ZP%nAvd-oG%OXSGBLw+-| zTK?GmhUTt@;Nh_V=0??azqiN3jTnefG5Yz>S|0xNpX1Bx!%2iTiV{EjSeRB=Zo;Yr z`-x|AGBhy69X)3aW}GJpr(+L$my?KL-cJ>$Y2#xQJq6lmmqB-ZSiwFHvjV%XZ>BD_AcNgIQ+N8Iey#9|HJyNi?jW`ad*Et zw$4u-w$D!qYgll0zr8mD7iarR|IvGvF3$E>#@hcBXaAm`-u*Wu(DjjiN)hK3j^Ir+pIL~tH>akf;moVR++$ei#BMDjnv*1Xp`OR z8IRu$j?2$JpNx+3L)}xdLwFm|x{l?>Z@>}htk<00i~iYbhEF(2wFud8J6<(wKMweM z(7RSk=@#SVls%O#@hsGWhd&@74j)pRSOw%q8;~|}8E9g>gW1r?PpvBSbcN87gc;^i z7WQlxa31sL3OOK6sBm!w(5Q3W9W8+d9D!$d<=>y&!d>penR7ZCYT?9HZkrVzPcjQ9 z`)tmpzI5L9>YA~hB!cEIlOiKaqqybK8A@})v$V6)vF4K5O3y9A>zXPqqQlLq8`#J9 z!dii4MB+*@m=>-}Sd!OTI zJvkxHV;3YvehN^s4OA21Ea0y3uc8#o&4%6ykQ8+jQ#z+RiR5|=5Mmz&e6A7?!W1&1 z+6H{MDO$htyIeZny%uO8yy#INkPUC-Ht!F{F@?F!=6t(MD7pffs_WKcMCF$8zyk@j_I_mlF$Qiy@gNNjZl7=k9J` z{BD$z7XskEDrN)rWB zIn27oPyh2(14QX;Vdw&{QJL?+Do@bLse=@CTr_BatJWNwh-dDWWXzZ+aqn0UUa9># zdZcjS8E`k_d_F}5Cn!ko`*#!%t)`uq-tsU!rpt2t_2%{OZb}`yrcR!d0K`j&>gaGV z#)CR_J4N{5+Ob zqNbI_eZG|rppYCQ*+z+SDZ}I*re?+fYe1C0`$*5C>-6H-eciPdIp_?%Ov@j(zOH~D z!`iB&VsCVXWWQUwljDK_PFY+77{+Ac@ObM)w z8pHcm54(Fk5H&x^#r2!Nu3t+BhkmO0ffLp!ywJK9Ri9f}NSDc7usQYV0q6_vQzo^J zOsac9dFWux$`CCYX7c4y)--Fk9k8l+?Wc8D7q+*nD0+;w*`8(u8Lc&kH3FSQvl3T3 zcLtGqqy?-FK0E)pKgSx8Hus;64BBM?E~a75qk&h2yqt=gN}@gcq?oSKC2BkPJ$m;# z;$k^)YFEN{8innkeq{vt0M)hhu>01M0A40qBy-AagM1i9aW2-1BJC#przcuiMAd#Wcs6L-$RpFpGwH zo2f*Ra<#2{DXX|_XjafAOb}9&=8oGf+#$A$dQ4lgW!70aGy0=&6qf?Gnxn`cw_2UE zWhd+Yx_kIf5Qj@=KA)jMl2zAbI+m3lgoSw{JDlBsmhon&)nIbsB52a~3ju_tP=425 z|9PFDE;;hZ0;YSn*Qv3YP$M2|AnIAbVBNwNf-g5_oNf~eay>keqz60wI$$NW)4h$Y zyd`0ozPm|c`fQ*exW&kHT`kUe%%D@UKc9B4omZ~q7U0-i`Zk1?WY!&D#}5CVGx5fTlQD3`&Wb&}6`bAJ8H>*AzU3*vb^i76 z{K*#)C&!@mut@PLso(+a3Pw2Tl|x}An1-CwQjVhKiRSY(Fnxh*Tty#WN8++*_nH(lHsF1#%D^;6 zS$Y{Z`YsmG?sPE>0%OTBO4E!vM+Q!6ACmzhkP-5&}|6@9JhODDD{^&8d{Zjj#(rmMaY5GFf~ zemuWDbAVrR@G%$W`OZN-VUnA`;VNcXBuaH+CI=&4-peX5AEIwxLD%j2US{{|c|FRn znpeWReXbp>k&qazHPduHEHq=k`vu$th?>#e)f-&%qylW=-xEQz!l~;>4BkUxwk!JD zE1$V2q8da^wvoAtTBs8R3tExYx&|?*Lfr-*VO{)^!c)Dh=8s^Sb8JEBBe#ktg z;AL?U0Jr6<$F^izxeJ9nq* z-1W?K_uO}NQ_eR$^+3svG3GKGY4L|U@W`#%^K=EbDPvKjcR+u_;sbTn(m z*(#9W2g%#f`Bq+yW+Uj-H^4^ijseR_X3|VmDE|77S+3GQowEo=W@}#}FB%eq;*TnD z9%x`sef>O2gOG%JbQw#*uo}id+;5`%82#w)w-e3NF8oerx)~&G~&FxHPa!%aYA9^xdfAh>AJrHdNf8a zg)(ThNls`&1k&kg+gw?%zKSV*sJ;-9zm0gL5sX^khuWku1yd}mvloxpZtI}|Z38++ z{hb1Jzd>5op;X!4^5%JtWq&=y%%lUho;>~5bR?XL-vD{qjRlz!M5y_cWkw9r6k{{ zCRIJ)!2Hd6nQ0dBU!vOR4$p~D%a5R?Wxxg^1lScp1vFsiQp9g>$;0X}?JOoda9)J& zqd5+S>w8j+C{T|U_}Yb`G#a^oy;5MYYGZ( zbV2T!aNyrPV50=`7x!UysehAr2L;B%7pj%0JVbnvB1;&hY8KuF zpoB7$+(z;M0GJ`4vF^#HQD`;s%DPJO*>)X!cQgCy%;#Duz`vXe!05|0`{H$C9({Ks zR3o^Yi+?^H*AA-hjDJ2IeRum}nAZ-fFN=RZ1yHfA6kPh**9vp56kG~^H+}2N;a|?R zwl#h0i+vEPYvWwb`8fRdihnuhAn+A=k~aG8rttqXl}3fyan7K5OyX5vV~W;0#9SOpbpO-U)b4IqVoO?7We$gC zVt%yT-LOGQ!a*t1-#{^aZm{B*OVBr`;DYDbT_R3%c?;)<%4n0$O0D67NEe1{hKtH8 zR$_P=oJy0=h$dON!lA1;gTU54mbO^+C22EZ9plHSBg=~=OTvQe=O(XS+i+KjUew2Ygk zuySKN-9v{m)Evv)2rEVsr6s?Oop$+h#Jx8AHeM~VNRn8RqlZ$UJXoXh_{ zbTV&aqd@l-V&qWRBB3x@AG+cxjC%P{iFK#rMzfXM2NVeQXFZ1yZ=I%OZs&WCamoBC z{+ij74eurM2S2(gg^QlL5lcTnvM5wYd^%usO~QsO?uwuZ_`@IXBWYvogldZD( z?91ive(k1)y3)N(_c=)+lm~XI#UW$K(bb98FaqeEGL)e;54eHleDl*W=hp72h^xke zx7+hzn%{xbg?0ye=yW8-*{744y!IqhvJXx7bbZF^I0#Kb@1H6OfAAY(kWu{mP;)lZ z7>n`-*~CN3{7Z4JI{#Mqtzm_>OBm7?$btb4Ow`i)3HNYnF`f9t8K#pxzXG9I`Xv#t zUBuLFcIPOeT6#sDwi)UMr7v-Hr3*sQ%OY`bu)i>_e{Q*NNlM>$5WL27%zz>OE5%k zo6#7mI^ZL~&X#_fPu5E{nZ2rmP5axA$Ao5e9XKs3@p^Y0ma1PrTX|Td7?~`%taDRS zjLY&|#L00_1W^17jqTKj(sYeH4_EHJo&AhZPAoP*`?~CUsXgYeR!rz$4UXgW+uj%q z+(n_m2Wmbs&n~Z_GMdEOk_g>uB!brR^bGe8LGpDbK@ZOwtnAQsB3VW3j;x>Frg?3x z>DHoJZ#E}QzK>+SLV+M+_OKrors}=k%TD>%vpVJJBYY7?0p%WZ8Kh76zgTW+ecMZu z6TmxYy&#ZBT)Hmfd`|m!b%5|68WGw9m=Errj1~_Xf`hy&3$_E zBld>zQ_{OSyU1mHIU$x0A8UpVedu9>n`x>ztpt>MPpwuvEHU*rR~aDBdHA6#hS79QRrfp813lHG~?;JH1dc0$x z90wEVu2EkE#+8qbFx}e)TCYAzL{lbyPZvTJg>O%!sy!Fy=qYj_ z06l%%m9}{)823+lW(`D{oembO#JB(3*SX)FH(#?2*%cn#F&ys4<5Sn>DKA<(l$*GS z%U1R@!6Lv+!pxJHGJG(TKR5>EScEp}>%<*AfStG+uWfW!`EY0;A}%WdOj$0O81cK{Xo)R&$)xN6yv_DyM8O zd#J1nHdkMl9r@EBo`9d1&+Ap@M)pXIsf=1ZEcW7p@g0ST4z1w(-c!mF^Xx`g{o$?g+r%_*|N*_?`3)zbFSul_xXZC(>Mj(+U z{}l!3p5|0g#)oBC^EjPGp*{8>?@W^h;h4bzV{?I--VaHjSBR>a&b3Ja!ALH3m#|D; zzz1+1y|gaFJ*i@BB!Ut(In$5@)`yZbHg&(U6O36l-D?uMO#)Z<%_!$Cz58Pk zt;tt%?jK+XZ*}N6K15w-c1jlI!IzODw2>8iVVv}S!TT|TJN8&n$`=>uEiwF;g_c4? z8~IF-_rMU5E6$t>fVEffcd;c-03p?ae`6{Hx{OdCgmE;2l2ddax9KII`xE~q8ZRwOJIGKI?gahf^5JXSg9o}c}OQx z!H%mZCCYKKqP@c_>*dc8VNe%bTKN?!yd{o9Y%KUg&9As}L3|aa=Pc|&&eEIHp09%k z$u@Fb;8#pUl8Zdk+qXt$-Tc43+!Ero*6i86clEh%?m6{CFV{D`nkNMu0~-!f$vkl7Yh>lf`xy4z2B}IoC3ezS8I%nzIY_fSeS z6~CrR`Q${27{;eQR0{;pA-|90{Jo%>aVi z(BHw|vL#UF3DNuQk6&%*=jbGjuj+IKW@iORsg+JXy!pPa^8;CFiP9;Qas5^+x2mV@T=cTL=`L9RF^mr1%Oco0k zSL1h!6T726Pc@imw^M+smMse}vdp$RU9W!!O~&+W7P1A0LG^!8Ovs z4=U@#a_|tAhYOJ+BGTM5{qYB()bBDL5Y@sD8d_7VAWDgMxFSXi9sFzBNZ!U`Oo(Yd z6u!l2HKXHIqn0wdIL+mGfxT#NmupnQdMJdYgi0l}9q-=GrWm3uF`bhF4-% z-Mke_k0n9pFT&yzBHm5ylm^V3r)bl^1h`9J1EyE{cx{k2q9pVubHuC`4v|72o5%6H z!}bnBxTB^9o_+1eqZiHa7ke^6tG~JXZ7uRH?yx)d!?>;txO0wf*A+8MW?(s*#lJfc z;}TyVh1Q5Uj;?SXee-T09(6B$3TmrsOuU$jR`_;*>0ewgfX`njPuqGu{GwSd;4fa< zxbX)`y@=lpRpl-~5UbM4yS)Z4Gh8OE2Q7Q=4($1-K`&c3Y`7PJzR~ElO|!fU%qv`) zf#oPx588rCz4+=;OSQFxU@AY&vV2JAl1hirF5Tw73_>dS_Klj=`F|S);+#z_GjKVGcH#U~z`^iy{PrWY%bB zc=2UAiO)E$(Cm?wnj{6w!imwUPDdyD59*nQpRJ*?21&bLs? z6N|M;Po&bAFC|&8N1$nXffnmlC8Q&dhiSdM%tuYPU&bW6Hq|vC$ZI)x1NCNa6zvC2 z|Dd!)OVCjt1Mo~88lNFnARIFI%L=!BorsaPlN|5rOzDD`?E+>seOl*tzV(?7zOA}c zgOqCRe=XEIC<*c?Vj{KPYkA=)Kv;B3YkJQ86(~}8<^w0o+nU-7tJSe*YCl;Ys_vfl z@Th8_l!J&z)Ukzyr*p6#a0C4UjIJQD{a3zSZEX~ml0J3AIv<63b%aa$`yux&2k#zf z62kpNF5Aqay*T9g0THrldJt1=^p705!!eNcP374vWTmo^-EcvCJ}Od{lj zfPB?TridfO2)F(kx;|sqcy%=lOSe@0aXc}@1oX&133fu(u>g~AV_PfwPpl_%Q0XJ3chVZ zT}6_zSSZ-CpzgT~-pqM1b3?TT6Uc(>^hp-g$-~lYO&2>kf_>WAO4C!zu*<{0H?7W= zZg))Df9>35Y*2leOrYq+aPAT9u0YTu{%X&Rnt9oCdN^lEwmXNfvZ)ARTgf3%yA@L$}tB>A%C~^<_-XsCkpm_u?`F~vQ` zm?!F{>6o|Ox(~h&mE|n-_H{Xu?UjBvR6rz+%j+kB;DbY?)A_&p38nuH-PIn8*Ze`qQd@p8OK}CVE8QCpF5oMO+N?r1#nD=Zs%+Xd>qCw{H#a z3q@1=42!BE7KfxUj?v1ZKGFZ-rFLpaUS~{Os6x+kA z*V$Hs(M2xOu^YkQq4T90uk%e}u}~q=OW~)KPztJ<@}j;qOfKQ@c)pdJ!Lm46r!9i~ zLG>9X?c665q@Yyl{|;t~It@BgH5+;^pwUUB=U?`i0H#)DAcfcWc*x>V0;dhNsif($ zu>7F#$i$??+gB*LYaV>_#|5-sL;?F`tyKBs~{E{%vPdk|Ae9B9Nfvs^DS zl6aLQ&n(l2HB2``VIFMEAjFNX5un$4I`91)a6~&HYST1%((9HJ(CDgh$mqRhGTZ=I z(-iav<5X?FL+6EMPnyr;Z6;!z6F&X)Q{=vor$^GYO~GwcHdQ@&Iom)8_{7#xI+cYj zR#^?)5^J*OFKlquK%%W^F13ZXzGg%F`{X}&RsV$OXp?{Im-{!h`BfdY^5N_x>>#NM z<8i6zq4yTshT?W4XVDs|)3HQVs&V!_FZ2h5Z;ddKVt)fJ#*Lx#o3ZXG2)h@xOI>i>5U6s%kY=Bff z_R{{aNh8)%F>#-h(#YKoOGo*xc(E<^-Ld(1Pcx%bqxXJRC#71v*;LoE-7upTYv00w zDdsOvz9zOS)2^OgtrF!;WG8&)3!V`Q}t~h&)E=y*x&#(9>^m(aeqjhUdGYeur(5#zg^k zJwzlS=Z_JsgBK|{%#;O?GlF5pYq=OCcoUWe%J8x7i+-MFWPR&58@(R_KI#{bm66Eg z!`BMx|ML%Yd%y6fj&~YJ22Q1On^k@fD03>&U;ur?XIC_W75EPv+w0@b3|i|=)pVh^ z_XHdWERAE32ZFZ5yY~R#ZY_6}-i913FQGgd*iv|zH4_a`hK_Aa8{0DTwN~~Ly<}%O?mEG$D>-Uc<*CTjUas|ox}U&TP}#+(Id2_NdK%s@NNd7q$5Ale~?Q_P@_ z0LqzrAq`S;ac!41qagV0ztIU+Baf=z$t8P!dR}jL``i>D8D5F>Dwv#g$UQ_pd<*gW#EVeUZ~? zlF1k`Jin-oOeB>6)rgaxAvOuc6tNa6-Ug#$=Fs0xLQ$|=liajzI+q+tSBF-Qr8LOfC~op6)!!2_m{WCz+%;@wpAc_4&&(N+SWG4#b{x#c zju&LiOD|L$`8~FminQkd^xij3mFVtT9-ZuB4yK9V%|yzz(F?P|~M?s>4`n>dJ%XKnZA-v{@uVSev>IPvE@kmeEpbG6|^7^u}R zj^yfm_w3mk)m9<8(!s)BWeM}1fs#SQI<5?Gf3(@ja8p{dtJY{LrWSc-^)IDv?Qno3 z4vshWU#O2G=gH<~$0GDG$x$-evVHuQh=s{KwV!TlVp(J^L~YFVyXSco!M+1m>C2ki zeqi9ZU_z07-A4Bp`?}3{+BFBxmr^O2fNT$yG$J+mOi4NJj-D0#=a~-CLCdWod%U)W z|6f`m`K=e9gi$C^@74ur)$!u#%5&%NgeVwif}Z*)WoU{bp0kF*2sM0LO}AgU8%XoK zp|4qV5qu?c1pVRZb3%v=Q4W70z=}JyyT}x1IC!Nb$&+@u2Mc_y6W)Oifa{3|)$NGS)AX&{C4 z!TjmYZ_Q)_3woBg0SB=eFOuFz50yNO1RRQ8u^CszWsQ-t#^I*$fx;o?5xC6)7DIQC z?IbCll_9Y5rNtA8BRdal)H+K^ghEs|w8cB~U@ZB<`Yr3ER3|K~k&MCe)B8r@ygb<~ znl(S>XW!`O!=Y|oOTL5D>F%w|0qZ&{8+CBFf|HKYKXm!l6sq*i2Xe7YxZM_6i%R-| zG-s=lnrY-^)wnM`4&*`ln2RLX_AMcye(}x*sWr?A|F* zawU7{esOvhs$Fw;c_2vPY2rvt&f_K}iqfy`CK${}B5xr*?vU091|ji)l}%l`C>yT6#c{<&gya#JsyXmt8gjH>_l=EnS;X_j8UWyQ^MHUAZN=!Mq`OJs zg!Hg@n#gbjMlw>N00E}WD29%5KB72|eO5Pu8ZWkS&JYhmW6*jKWnXk&5{kf?9VEiG z&_@sW3U=nl>s-IC_BB`FCAh#urSd4U}|p-o|R%R?!Fu8DASXP?QfNL-0+8_uOnBo3#isQ zWl;pDZ#B5cW>t!m%p@kpaLt9Ogu?A9@R6PtaeddX&#TcQcV~FAP}=u|X|Ywe9YE6nZ`|KC}d=_RnxPqkP_zZP&z?;phW)aimI$Ix=gUwlBo>xsuW!J=rUM zGs&H)iZs++T-$m*eV|^PX64;vmBC`!@3LkCTri3dOHlX2$yJy$%{bl?g|j+N~y5|6mp+2WFr zmx<8Odnl(!5+Xc;O-5=C_`$1bhUj#qd$2Qa!aA~(061(ESbdtJ?3n>8`nVbu_0tSM znKA9JQmyA?XN&tOT*3+5G;0DOz4BG4JLAR*JqQ0!2;2m2>pr~x2Nh9l;?!r+D?4@H zAy3zkr22=xTPIj8eRdE0$e@zh=ESeb+@NCeg0V5d-wD2fFwvsGUby#W5_}Cy{2kOH z{P-dlX{YTc{pc7<@`BBUpNCW7pns?1O@$2EuNN?`zE3Y_Un9TA5($Xd|CE?zlTy>f zzzH7VFd08JPS&Kj<`!Sq^p1(rb7-#1H6L!G${kt;OLe0~c#LXW)qwqQDBQfn&I}|k zt*UWsw_ar{IpjK1(P+bVTEbUl(nlKP$a&JL+NDb;C1Ard-*s2(1bDP>2VaNZ>z%aY z$BX9g8vkmioza>lNnUW(y<5v~Oj6{x2q;D**0Wqt0-H`!_PBq_x|^l|Q@P1*Jmj z;s@WQ#XK*@0$9_r`;2noJzJ+N!G6Z&Lo82&_2F!)9gwXUS45;Smg((1#%?;DUXTs? z>CUt#`_(y2N{wUWrWj>SRDGz&IrUtN1l=RcpQRuI@vYY)G$>DxdQ?0oEKI4I`&Jxl zBFWpJYM~O!N_E=mk@@ZA7Mm8o!2vW^8DLns&2jat)xLv}I7+EPnZi3o%iS28KElZA-T)|LWN?}}n>M#aJ&p`b{xS_GrU|?<#@2O?)j?HMU%TEu+EYUDS9+8OD zN$q)>9YuMW#1!2GaYEijCX|nVVAe^JKlybhs*jwsGJ}f1x^wbi>?emo=+*Z*m1gQo z3Ebi5ItlFE4>`4PRM8PDIoYIh6wxZCMN@Ct+`#skEVSh~&Jq+H2QgUCv#!yjf;H=9 z#GHsY`8e*;m00dQee9b9XFQo=h63fMpk>rRRg&x%6y`@7r@_pRadn|*4+a^@U;y8R zRESR^$>HLmcjufDm(%9tlF8)%WqX$_pt=_$NWmsM% z(jMoWb2Dl2ZtehEyj9x7^cVGJ0hy$HtW(_U5-DD&0*97$a+GJbk7u%tC!IndY06FO z<&V@l8nJ~`oSRER`Y*4O4T8&+&m#yKrjC6MeiK{+BZQBaQIYrS^xw$Kr%VksBfe6aj3?Hu)oETz^eMWg2dO^uQCey z@4y4}yP>pATfRxK$hlV=C&qg_Xn^JGFbXpFTqSdtlU)VFxy#RmWRy&ZCx=7AlZ@LK zXLQ7miQ(rD9Y(S1Y4w zJdc^R2njUY>Mp8bMpZf9)p4y8OIvAx_VL-y|Slvx0oF=Tbx zureJFa_&TZ4)iZNLI?9Ko4%IKwiZIP`3sS>QFG6uaCO-J*%fmMV08g3CAVhpn8`c0 z{~*GKLXwraDuam{2DRBMq})}9%xEPIyngJlh@B2yfBH9Vk44*oY=~8)M?rU>el;Ry zb{W(4ZGxRAjM>X7vJ2Oec$l@vPqC7B8;u`aO(^zC>{Vf+lT;PUwh}243AJ7{f6*;308h{m-t;*V5YPpBiI!+3+`(1LfvHpZrijn=?{IoXTC%qqj zDxZD=PAlw1pI-}TN;;KC*FieVM*KC8$g6qsVPGv4N>DxLi0>C{q8CoG)OOK?Xp%_n z#Jtll^uAfMy3G;TJaI*{vQG^8t4=O?e^$Pu?o}K4$C@c!C)E&TySvtK zCF+DD)K_svxX1GFCJkW&B4v^Ncd1x5%Y907k|1}~a!^-VSWuG5b~g}~o=%y~fRxZp#dqZOXrU+AF`Pmr>>1-o!hBYQ z@4Tgxx^BZ6P55Vh-udFuF9N)Oi0i0eR138-;)GrLnv*oN3EwiL*P!VKMtyHuDCuom`4`#B?0as!8oN z%74ejkvLwf_-qh$Aja~RvvLSPz(@Z~ZoLmrtQX}W;>GF{Ap3a%XusBv@7IBZ!PMnJ zj-l`#^nAoMNsiMGmY7)LN?W_GTCCX*$Y|rY!zb0aV-ioN59wtzlvW7m}^J%Ad1=TWxT4t=_9*Hra%UOOBPQ7U975?Y_KmB*s(C0S$ ze{lJce?9?#26R{R;HwbfzR>#k(KyRw$#ppp344)x zEkWP1mgY+9&U38kx5C>)*HuwFD(mZ2~eJ3oCZw%$hS6>q;_{y!gqho60i z0e!6oAdh52dpV;oa9Oi@EXm^%e;Dcqt1SrT3AqF`mWmax!5l>>i_1@Fp3)ztiG2Q0 zY}z;tTP9hDD(Av;4Sn%db9^T9d-%KiEzymQ)}xOrP5XFxmIqj0m|4z|EeH)k@r;7; z41H@Un`NRgPyB5C{1*P=i0_J2QAeEnn&2l;#cf6p_B{W6|m#I|t$-r>!% zMMXBfa#y?-E-&b(uMP9KumMK;SRAch86$|t&SkFZ+*oDN(SDc`*TO>`i<;OKpNg7d z{t75!KNIMqXStcR3eZQ+v;3Hd0bMfS*(@ptZ%#GrgX&P86&ZVHQ2Ce9^RaPTup;Hy z9sv|IxsSeSurY^MLM99z@c$HcmT}ys8f6J$6;Ah!6#=U|#vpL6n8Wng#>TzJHZnjK zizO?5&rtqE54WxzG>6VT6Qm0hii0$GL=N{;m~4+<;waNUX#7U(xD2jl{Ax z3J$$`w+`U4?2Vm{Z4KEcko`q`FvCP{uTV^xtcsZHJY>4J{cPf#?a^HR(tNGLOr_z( z>XzCJ+J6so{{-KpH_cBucF@?Q*M-hiiZPDXHQvNY?ZDDw zt}=UF*5ik2(`B#@{}p@le_R@rS?u7%gyFBa`>2omb`wA7%F>y{hUe!qhli_j?Gkh| z^5c-xB;}!yKMl8~nyalXP=02pU3E##N@K{Vc~6ZP6l(#^h}iCUQEZL}Ur7&+2_t39 z_-_`58#MJDE#xf#POJjaN0$z@ZxFJbGaf!SK?@Z~%++sikI0Sr=G_z7IHuCd3ex~q zMACo*U1)l3CNLlzdatTsEwfVBqVD~+$HtOPS$1FCa8z{$4C4V=9%TqD*o5)U$ zU43_{HfIJdYbE!+6B_h0dOK%qZ82Cbk2Y|Mz0JJA9l)N5;b~#Qe}O}sAU`3mH)|6_)R^<_VZga!-xQ+h)0Oe zFX;X?_En17Npbh(gTBn9Z^oW7fg(*~9Z=>fIbD_?dJdP2B2IVfOB(7vnvYqs&5(^X zk~CSyr{+2cG(_LRk>AEy8FMG-6kdITGW*r;&ph3NDJUVbuo1t*g;j7SCg#70j!1>qA5CdTuX`Wn(NCeCh z5$Tj54nszh%jeW6;Y)uDkucBdEF(0-qDP8SXme2O(b37*Sve7YQ!DM;@YQ=ysMs>g z!i4^6AWkOty^2IOH635?j$Ql_va~#yclA=NLQeItQt?_U%Izi-U2O6{%BD=4YzA(n zJgRcg|5;H>^iNdE)A%(s!z|{OX!@s*3;|k9w-5q98Z)>U9w{Q#O_crXe%vUx+2pyC zyBXsimL9x915sL)s??=fcpTsg+K=WQ3c(wR*vC4M%>?fGr#13bhm$@=r_3@Z9epgp zc5EE*(%p+uLOY%h$w@!WI$70cLirSsDu^L0_U5|3g z@S9sx;Pd4P!dmAX4KGZ^uCk!rV3>b$=rL685ynYf{SP0h(^>Ys{FLkzx&xG?Q*Vo~ zksNS?P48v>?iwg~Vv_iwES2oq#M$zzlB5P3m>PX_Xfo|M`~eEwku_r<{isJMR**2I~g$NK{Gx@wQagGNq*L z!eKK3#@TC`wit+)Umx1vu>8al(Dgz_O2}Une@n zg)!^6EF#}ZUpO{goX>l3p7TYxzf)h-k(}4-1a;cBw$NGnm8_Ih(&=ZC#ZN{Ow(B!+ z7EPoQJes(S%0pSfnBsHrN^68NK7OzpK}eu-{0C=Hj)&?bJ%|I|F`m@!HleCCFc7d1 ze=aJHdc!?wu%P5coV7(~E)r?dO>FzSGq@{buR&+uWdw&-W^LuW@vg5N-0lweopuc( zepRX0vJ&}_=%W`Xt-|)eb{f0|N7_NOe*}BQR=Wi5D>U4l*>D;B**p{vc}RIg=a+qh z@Eud)X~xb(_p3p`mLt3%;+QICd~leIl$xjpZnoX+)Y`Cd3vx7K!MyDbKQgh67t{C^2uF}&wK}BxgUKevO`d1b3 zw-l@^5TP(N(=lPSsF}0?r^OvYD3+P_1oW9{u&Ww-6|TO`mE5faEIwKyYGmx&h^>uI zJ=fnO2O+{2w$mOYlW0tQ^=WU%t4|%RliB1|j1yPH`N>NMpL(Dz&bKQ3%#uBRH&26Q zbUh^v*yy*T)J#g4kws!=45f{x3GPHfu^K15L~}~MV_ER_x7WZ(Iasm>yYzmY{;HOS zhDZxyw?;?7}pEla+=S<`=e{~L{K#xokg z_e@O#e}!jS#?(25i^XzZ#%p%VAk?<`s>)vrPpf{tR=t(`1A(X9YW#StJy$*V(FTEv zwx*3*6Nz)~fXfs;RmQ8$m_e7V7t;_8e$1)o8fSwr?v@6*lE(SiBB$n-g{{cx^Jd=roWj>BRDRXZ z)>6<-O4M=g`y6Cr=KI_;gUf|5_j3Wc+E3?;)qfBl>sR@=8{yni5R)aT+D~WEYWcUD z+E3@;VZr;Hhk)8o=d9I*@an@S5+8&7*J;TAR5|Z+4!vK@jPwd$ryKwK$MHh=w4mxI z*YAJ&i+_aHQm9E9w70cDJiib^Oi}$rtHfzv112naP?fOg_94n@U-NtnPrHLzpOI-& z3_=%^E5hNfT3s6J>dXiLPDqdMs{%LFpL7b`gtRcuUi9tRiF;?qsYT~KMYbw~9{K?? z9Tn2d)#?sV^dU~OknxZrwFWnUEC=>f9I7pXf~H$H*uIHAbY54#bqUT4>PpOU>LErL zm8`A80~>o!CDf&+5^h+nU0mc-W2kc$U&wQ#VfbbQMh<2@a#1wEFw+CmX_XO;spD@1 z1+LXl2c~I~P-X>P4fED)%VxH;XarN-sB?=gnXvj~tD1{%m%EA)8Z7ePTZDwakHopl zU$mK24C4mjUx>TJl{QB%AC0KF2C)kD=YL19L%^}FRt|{dGLJ9Zk1Txe4B*xd@XI+3 znt?={epe&a9%?Mpt7D)Noj}RMF-vDHn%I+$!daajf6s~zRs=OFGHX){3N+dPN_gYWIWOcgS>E1Kx{2&q^^;6BWMn_V-eLI=SYtITLq=~0N!2L*V z6A@wY9X4~{J9iI_INgI2GfA8VG1)$-t(}gsBCvK&1ikp3yH~>BD{A9pL7RXPU}*SF z__2dC(>y|2uA#oMPUKuEDE0oXX5dM^w>?=qy$;)b+(f8}Z{$5CGsxxwF@oB+jG(=7{mO_L&`NYwSnwMI-2F&~PBg zjOE!9X(7>KojP9KPkhwe1R|r$XfTfB18Ho9!;{8W*GpBErL40}{acF1#Cnytab`pO z_3ipki+=rF)%+;Qr;CP3hDL6cij2+ZAyGX>AZjP&j7~~*3d>Bs!*K`x#*Z{>#;?{uq z?eUJHG#Wr`B7_N=|2YnF@?^~fuPM!C-;3iM`-U-nMFK(76@~1~^C|J=>p|qZqr?u7 z#xP>MMVASnLtn~?m*`Im(J+XEmbCQ*LJhNHA3gp%WiDQEo=sM@37ncWSl=B`ckSXy z(S5u`I4FywXlM+)e+i+t6Rg#*W*PJ_#{7t?n&J8iLCoAJi1-J}z@53f8S{^Hdu^i2 zKmORj^c&v_u`-s%&Bz?{+dR!5i{sxGbf?E*Bb3{Ol&0|S8Az=h>kheb`$KO+n0-I5 zWo0*v;_?i7y1(`)ziSB1FZVg%&K`7kxveDOC*x}?sQ}5jD$Z^I3s%YJj^) zF=jHN*S=DRC-#qMCKP3Yx+2)zy|T z1YO?yb$w3>i1wAOzG_<)@J9sZe=PP3ms?fViXLe;`qsP7)&1OD>U?9#u;rCNN^hs? zDVIXs7#ewe9t%0!2ws>#E*9Lztbi)6n^1PI7S@!7sa4o21R8v0 zcP(azZT#c}R|8j^K!5LL+`|S2yNsl9R3(YUB(1Q^jcJb)6kPjDomuj{?2%-N>x` z3w78FT`E@*O$Xu=#m4898geU^ued^M?B$kxDlN{A&DE;rdF-K~V)zYU7FZ*N1kU^j z18e8VcM+H0}DH&4xFCn4hx7bH{%NBORy$)MEzaNI%hKx0cCvCT&BX6aHVh$M#Mp= zK;$IxH$ip&ndbOTfwit^Fr3!`)h*!# z4(5&jAEW;C-3zb<`=iN$5GJ-zh)>N9nDa_erOWcrMtR?)QB94pV(IaFhPwQ#5dSk?-pJWoI#?7bNXd7QWa0 z#71oJ^b-u%H5eM>w8^C>AaD~V02^JC&1h_v-r(WLBMN5^DpYYEMhL2rcZshs5MT)3 zgwrR*04S!dwQv+R&!k?#zB#lMbLYRpB7M8E_#JsU`K(UL(D4nbKi}Q5>TnhaZff`) z?!|r4AE*sHiv!c2K@D=9vf(qDR!*Hm&qG1^Cr!|cjcS&=f+`Mh)C z3?SyhRNk}uyF&9+d{$X*;JG|^rmIy5;|MY6A>MJ1)w??N9Adk&I_Tb+YKO*Bg6tBk zn&R`gIB!*`k)X&(_BoM+*9`lmC3rnILAwBX`#A~Ic)MsERE#oc1>XOosB|{}l7^uQ zPP_j5HA{M0$XZ?VVdDqu9UJh)0MOTqE_CC z#%h~rYQS_iEH{i|?lL%$eT0QjviZ0#M^F!zSqGc!2W^a@>_ugnp7AV#@SKK*vwmC> zhK9&$TQD=q0ibSgb89vv?t-2{mfH98*G4k)d09z@@OkSm0tcs62hDU2F zLRUitp8P=Oc%Q*?Qhpjht+Gc&Cmj9$Y~oP0 zCtS=0PJO$D(H#Flw}*wDjxr#64|Jqu>&I2IQ}IoL#^vnDgF2p2bEK9iFgJ1^gs)?V zAI=R6xlFYof6pqOKd5;N2ZoQs}MGFxl*h)`^KqB;KW+9X~4G#>7PED*>|iFbP3s%mAZ|uxNrw`<;h!_cA<=ZCX*H?LTCn=i%EAOc z$Qc<;kxYtM|DZv8hJ*=|(d&>yk8eUz&0nYMwgKZWupS&AN*v@!t;~|XC`t+uJtI`U zy#Y}zV4Be#WQiPloJ$>q2E9menfW7^DyeBTbzyd-SyRM6#73azQHBHNA9PHFyBWLB z@(_(a(w)IM6T+@SdUb9(UXd@II#auvA@Y@9yyF)p!!O-oh89Gsf^12mg1 zjY|J%a{bL3wZyzmQqi*1bTYB*%EP{$SEi5!oBXmKBR(z&-Oe1Coda{s2$&gF0PM7P z{Y;HyIT89t{$$PHP`cuczYEbwpQ6!T9Bec%XoK@Ev{qN4!_}4OOnrZiSww*QAE|LA zjS8wb_&-N8@o2NB_{D4Z+BlwAvFBQ)JHKSXTmlt&dh@}PBNcTd*6^8awlwN%YoNvTso_l7BBWscyCSb?YMl_}zv*07kg*HBBEr&a2&@f(Jr1?b!3>-8^4h zwD`+{)+%oOWhzSWKZt}$+XuO>K2s%gGZPAxcm zwDS<8O)lOLycO8}UVAFG*o@H!40{H_=-eqKsT;c5hBo4*WFbfY(0y zBn3PEK=r5a!T34tOd6!mL_n8gE0dVma2}oE%8w(tA+*nx8 zg{U0wDERa&tCxmc6XyG){H{4 zNbRB2_4A2}rW$MPdqb&%Y%p?s8Ya!qGrRwW1|}sy=~fbwR$8U@v}U;sAdkQr_Q%2# zwM>vb1LI!`K~L9tj81QC?&PO9%{8M}sqD7E_I^=>v4+q11Ti?X#9H;1da&#NJWlLv z%0U>MaevXa6e29ie!|8U->hd%ZgP)_)_Vy%e&i{6(3nr2_3FX7hE&q(2k1%4>4M4= zKXzBbtnS>RBoviS8Xo!gUf7lq-cBro5>rN|(6yDLdBEK;AT#W3+V6gWG#<8n|b2gT=HsbDImHr0NUni~5^-L5_a z+aE1@aj<8s${qB9k9znv}n-GO202v)C-l$8UqRSuGiTlaqTBW&NU3WQ#_# z{(qNh1ntA#BL7{bLI;@z{OPJ2RQC`BTCZjQm}~uv?WTS}?nLRlZyhjG-zWldb}Yjz zTmh}%;ZX+#w-jMCmkSz5P|ak&uZqS*ksn?}tNjxe%K~Shi4nE`*Z9W06Y_cVzH@^? zhU=WxOzgf`bqy!6Sv8}<6?^{AbemvoeV8bZCttgEp+3SDyQkvJ&~q@&Wu3#d`t~OY zZDWirZ#6Re(G2`t`mgD32c<^fMaS%@!50m0}XoNBBt~1+4jQ5Kok&$zd6@|naiTtnryMnoy=jVE8EwfT^murQLRazq$hmc z{=$Z_q|tFj_MZ32;Jdy1`#y42*Y|a9;B)8lyz4gLS2o~R7x+69^t-aZ`#i7;;fL$@ z={Zh%_0*!{JJbXrsk{F=y`ve)wa;ShWGi{>=cT(}i1y3Uq2u~o<=6_$0}i@Vt>cD= zs0w4L78X36=F|u=nQrFT;nr?3&zEF@lM-a*&zo8K?@#9Ey?pyaj&?bb`m9}rXD;CQ zF@MEUJV03r{a+RS7j&+3c=nvl*-STPxN?0b3V)1?}{5!8c!$wA)yR0JHI_ zeS#t%7EgbHOlt#cwyO#vS}JF+fc9b#^-_psY?EdUO$c(>t;@3Wp0!tBN+|(g*$jRY zf;W%X&B@oz!$T}x0tT?T0pX%*+nU$%^)3Slw(uE93$(2==>eK439$7A>91u;w(%L@sx|d9&cTRbdX(hQ_(zOEshmOEo1!`!%!USurP& z=1ebvS|-0<@#FN{UX7_~159H0om)Nx%In?^CL-Q$``e=QKd^OlmS$48hNy>$P&<-c zuk@LdJuvw<$roZK59RB!1JLgZ!d5$|beJTciaphh)Os+N0;QW8f91H( z2h(d34f_^X)EIJBtEfYvH(3`ta)6=^J&{lty*kJdY_s7p?A}DM_l-M8!1qV!md`kL zaTh&eS)(Cpdo~TiTBl$7U@zc5z>~`pij?{21O=K{k7FxQFWZz4bsvMasU%B`3SCA& zU9z#14<|an&3WDjChaw>V1h*}L%F>+A{;B=B-0zsb7}xNiG?YrcK!$3fUdM7N?Bcw z6Qw563?lod?76_-aJf%m$01Qq38`FPUd{_K3*2ozoH5I~YQ~R_?td4lOv9O zMkm&Z3OaK9w^SKTJ$Wc(FpmGqy6HVqnV%O5_EKm#V%ep}t;@8Yk|wf8$tQDg<58VYcypiWhA;aH@v-y=wiL_5-;e(={yyowve5Sq#* zPladQ6XsXv1Bf2jOQMb8Y=o`e`~!ie`!hVhq)U@I06JJvPRI(>ejq7Uk)`QcPeu~URb3Qc100$2k7k?{PUz!+khW43rDpP+ zf@DLq;x^JAa=sUVM!CLrpAE^d~w3XcaXt6!bg~pwrsTIWzh^~pgFPq;yLJA*sP>uZzRw(9QY~nFh zlX8mEJposjmzFH}-?!s$UPa#naZgBs&Tq6LD&?F_{3LC}k&?pw@Qz?0indW|KQ&At z7OGUrcSh(KMXO-P`Ww>m)j9LF|-P$Dy2qu|`}x z!D7yKuI^e+4?;EX-%|P1Sg^Nav7tP~!e2^F0Q^up&T3a6!^K`?ubh6n$oTrQ#FI`s zxiw$XB-v9TvPdN=AOhiubP(toiXZ2eP`#e?ke7^8L=~{bEu)8)gGQ;6-~Idqs)Ba`(D;^ZRgi7R!Ds zfOmgiPKa=Z?9|Bv5kOGC`$n8%Txe(BU>I?Zw?lv|(^v*L2hhVZM(9sahE%$M%b=bsQg z5Xp)hA(;|Cn3NJfC{BwWEplKbG^jJKIExrfiL-%;z$l@L^rTSjM2d)MBuW&sMTlr5 zOLVJ7{9XGL%ttQ%7kOCip)D7UKJ!@NkgCf`J1(4qZgQZ8()M7G(Z1Y=-?riCDiYvq zq-tCyaC~JHU&bp?Gb||n{;X(xrT*K^9x-L&c>dbM+41z53n=05cp5TuJohW4_fhH~ zoh>i?ex4b9ACvfMh&%9~e%4z)wqV>K$Tko8j=?$=ADY4BW5Lwf=u!4h@G@5hDFd!K zID7b!5NtaB3ysn-sEZfs)o0-Xf#)dpJd%$M^N+y6SL}`r;bOnBTL27k{DuyW!x7 zPF?9p#u>X3ABhxd(Rw(E)0=$NYVZO8pi{-IlE2aB`uSde@B00D6^=h=_~I{@0BpLw z)dk(~_;wPO%uV5j`qQ%o0ku{BoipgXA(5}XrT?SjIlOWqbaFV zYqRab#rFlyyaVmXp}A*BUUn3Pav4G>1}^0m&Yd zsQT4D`m-xSNdw)_20ct5$~xQqEF;UZ3DEl{%EdyTy?mN@88Qwoo3nPb8itWoSqB#% zc@f$za_kOOFFvwcmXTGLe$N(%7iZ;#VP;F!-QS0AkT*>XgI+62ggtI%hwJ*N+#75o zR-Jp8@GdIWtfZ(4F14d zHcvkaoyE&3hLVD_KPWg$#)pg-p)~|Bu2-GkP=eo~HR2~YtrRe}Hw zue{2^FQ^50e9FNu6gUMjiYh)iivaR&Kl^VwCRCXIp2UqeqemW-;2<6bk|J9M-O`NP z>YIa$i*cSk+MX(R1M%lZwu;Mungu5IJBpEM1+Tr_&`h_NM}&DrlohTyrA3K0{M|V0 zNKW{1{W&Msi_-JFaL=er2D`paZtK?Nr|lf7f}gL{Yt;-ABN-gL3u=dRufL9hF``Z1GV3?Z^*Z?1$-B zQQ-MvaJ*^+#w!b^(ei>J8z0C%fi&;SI<1A`hlRZanOcgqoJbES{qpVK1o@6)1+@ET zs)wha%QEao!+SU&l9J-(8vm%ahLrhG@Yy3d@Z=q=ka20_w_ut92h^t0h zIZn507+vHpuUYLZtaC!;L3Xe8zak2Myi4+ze7D^Un?Ed3wx069WeX6*Yk}J;o~|Kh7sFH{g|I^u%-v`^ z6&kvWXp1w#?iG+=yMZbjC~vZWvC9U?DjO`KuCao#ha@oy5)f9s%~ETE{R+s*`?d{A z>53npVl?$FtnpzvfXQ@I{abnbu70REG(i!3tTTc0at3)Z>svU%T$1Mgo`7h`Wf~Q-3d{Com8zhh zh>J5wD4Q=jP=3)1trJgbtTGMT#o^=#?wpXk(KrC-^B;$l=`{#Ff^4aAf$llhG*v6t zptG})MGQ$6wU9jVLuahwmRSsb&j)0u|D{D6PE-vibk$(e5NAJR;q8~h>GCURe+Ka> zTrdpUGqdKFmS(0eB1lPT8S%B7vz4J5lqsk&&yGm5tF;A-0+J%+S(DOD0wZ8z*yyr)^U=nB;ypfQ!y({`3c6L+HIkSs zoz|gZWe9BZE)I}#H5awKjiCqo6$e1-t3-N1C#>ci4y|pmp&Sk7)P&YjXXwyhvmFIW zq!8yxzyDblko*3FKSReyP|VE>K}h=a;Njec$^p-rUal(W#JyOZKM_f43rtTJ4Q=;G z_ZTfd9dVuRw&YslfBGFnj!=#3`(hU}z87S2F>2_C=GdSBfTmC$hJ zKGO>-9)#y?7*7LyFWTDW;~-p#@9dktD|(nl4t!rTKs=ISK7u)xFZZ5*Vr7reslqyf zpM9^}+c`kRsQEfWD*ANMD+bcjEaxm^1pkX7Zhgm2m24e3yFptY)8XIoX3BJl9w*pAa?i4}^E{ct zq-tc%Ct))ZFbIBg_9Qnoy)UR7abd+gm+Tm~h}YvO=arbPEgKU{8EveoJ-n{k+?CH_ zGA1ZKF25d^DgNxfR)|gma!oi4z^DWk4bPJx33CtQA7bG&Uq&`YYC3y0!KZ>=<-1@! zX(A1y-k|jGZIKYz1Jx+FQ?J(EJKhx+^$dU{`L&lrqU2|;Bzca>!d-^Rg1j0zUMjg! z#Ns-7SXC^GHpEL3;hsWK{Tf&Czs#4U=HUF4LpQtKUhdxN1KvDF{9=raLn64XUmUVW z^@CIzD7HI%97SZ%%a3}ybW@Vi;eD14Jrp&0IopbZCvM}+tN(B2XIyR;ENoJX?_v1k zhPb~(95*($W-y>S9+w)S91{4d!N#8_iDKObPu%?fdY_r@Pr(HkB0Cad-Cyl*z6TFk z87duTD=!iK*YDcou`y6oUFD=|`g93-Oj#ECgGKP)*7p*IiTFjY+_Vmu1V_;~FU>!o z@4Jfd$Ln>IBY_pWuB?!qA!B-%~y~5C!&D3l^8{zA}f=1nnKX3 zZZPLVD}B*(sWnk&V0YJB3wQW2Lg~D?jZU9_>{vSzqC{CCcU1DM3lojRY9uF^H>yIu z&=E<>UQegWfuHSA=x>r_+m|#97_#u8sl*wTx}QX#3pI?uEXLIgr2w{CLZi4}V1-9m z${5CfPIs++*E_a%bU$E#SN$WLIkX#JEtV4YF}5kdIaFRT2v3~YDOl4={Egz(KB-cS zh|hb#6S5ABp1^AO?l%ttapamt!N_(S1J(*vQdhCGprE?`S;p-Cp472p7|##;DHcFs z+RO%u+b$ijLcjYlQVXfY z?5;4}zHzCU`5`FR<{jG*`^=-Ezx33Hl}zIo8wv+o+$P@rRjbt@e;=a)gbee(T<}%? zArUcn%$Q)6#dW#`T&Y^5P3usK-1hz`6JU7;#W&%;F7r-CMY~eijTTub?O*-F{^hP0C0bQ;1*)4X7=aJgQ9D+ z$|vDvQRx`#Cg8jFq;Ql>SUuV+qt0or3E-8nVV2sK?zCUkN)zbfN68G)7KRoTXf z3NliU#Kw69%gHCoQ2dC<yh%@cJ|_YsIk})dbN_n`nK>f+Hp&eO z0Y=j49n9W*|DBa^Ne8fX1c;@T#SX%ar1E}SwxN5#*M&u+Kmhf6RLrCV{$0km(r-Bp3NMFmx#|cy9u>RJl*Czj#{BImU54Wki z>FnskT>fi>pbWH*DzNe@18kxUr1>TTY?5aVgyo>9VmJ0w_d76RXiSOj??u*fzkTX9 z4hi9=CzdMN>0Ba+H~sY9tEySoSvncSQzz(q#5H-wHaRDNnzrQuwZ4lD4Q2gAAJUMC z*2qo7t)Rt{JH*<$w@9|wlFua{Zscks_)V%)Xqs+vE%zNi7frROEizzs>N&2Qz}|N2 zd&a;uC*%+%&0kP_&=_@zuz@d;C?(?-qRLljwoBOVzYa6|}M^(tQ?Rbjp4C`9@vv@MuJ* z^q(61S^nYKF%B;B706SmQsb3qd9#vc<**aqoH}@Vvd@m@aWJ~*C`x^V|6f~i;X+fi zMjE#`?xdId29a#+a+0UCI^ujz6}xak6inb@r&d~)O2{q-)I~LEVrR*YxzMc|&J|sX zs~yVBbU{IEG``vdsOQrfQIgQ2F`p1=?H9Ii+|%cJ+wEx|^CpzNu9xY$CJKHY+-36TD1~U}%*^W47Soap zie-BK5Tq0ED#F>Fe-MK!|A2qjaOxzSRzVLMj0O0~exp~dVD6}TzAmSFeuXwIPIF7J zKaAy>N{j3Q(34uOV_UV*mSHq(=!V$L4@kH<3DeFC7{v1mo7^jUuK=n@C+q~}&O~-? zMDx+A+c{q+Cm%0fHihvhV+n!}|E;hPTZgfx>2Dh*{F}!~cX+UKfqr;z?*t5(IQTEn0CbX+%&}}c z_v+Q{9><~}ByiJ#f!SuvHiv-`Mycm;0xn&K@`U_;9ow(<@}_FA>+gu+@`09@%N#Di zLekp#TSYI}BYHkhHa_9h9Ea@wpEe4TVjdVEu_U)+jDOZBE-P`dkw$E;>Zvzo!5qqB z`1i+jO;)SRfhQo{UOZ#y3k>RXi`glerJ9UHw_`*=3TgVlmqBtEmzbuYGFfN0jp7<* zIs@%$HK;TfdIQ~L?XYJb7l1t4vZ)LHh{CJSOzUM@n&BY}*VQ){_<`L$|M1^cL5Za^ zTH3@N$dqEduwvWsS2};k&pJg90~O;M$p{eF*iRO!8RLY=oYHBqsU%@{zafve8ii{u zYYKXxddYD01XxMQhRm?#-K1HEoKTT8-`avlt&5s5!sLW?N`{IIgAwxs@kcTD4fuU`{<3&IOCJ$JKY9ozfcwLwkBn8B+epRUx^AR%5^o|JIlpSGsm~6ODj<9Xo>?pk7sxx#0=f2`cuUP z$^N~nwH}k3a2-2iH1(#ql^_R0tx$xO;l`2lr($!K*#0=ET4wQz)f0AraqRSajKlcKyMhi@ zjsCZhI>av)aHsc=QR5vbde_=zqFoPXSN+9v8-7y`=<0VCqwk=T|I_1Va$q&K`pWP3 zGE2Yj`}tv2y)PH&apH4U4zZ=&<7?EoBPe3d&-hWF(OgPtb0@^?KsKKFkpmUz=At2pHp59Sq^tQG0nuH}<_l zZOr>XE{x*i;g1I=Qk<1?5MO5KufU^NocVMN=MCO| zMjakZ|48GZP=!4R+Y_-{!{&Rz-+N&&db$XpJO>xCz341#_Z!#Id-uHeo!qm)568KV zbs=d8HAiUC)dZ_lR+u^-=+c|@!Zx(KcNu{}%IP#WX2(O)9J878TKw0}9MDAoJ#Yx>g~ zM(yN~phC~H%Yl~f3CQ|Nu!aOos3C=I=4LtoB?y3RDa#Wjk8HBXbvSK)o+~EDu>U^_ zB%K&Dsm>(MsKp<&n>#Je@)BklbZE3%Q~ZVr>>!Nj{Epshfe3pA3+zfhNXyEoT9b&n zNtV{I+QV?|lN52^O=PFu-E10&ugoedRQ;KtlJ(4EP z2tos%bNMY3BE#c;Ru9<}f1xdk;yQ|&#*0;dKXnu1^i}e+=KaOlzZn8#L4qa?*b%Da zHMjXY*SePV=rl3Eoc3!4@7 z;e@dYK}E|VvpV9AM=ZL=;6svt(F`A+-(xx_+{}0#4Xjki=$#i8U%YK}4kr6QNW6U| zaNA`BRE;RoJb>jeo{eHUC?rnrm zpzkA+?g`Ix5YfunV-JDFrF{cza1^Tc7AQ7UTYMn8jD~Y#z(pfgv2#R{If7JK`4DX#!6PLjnGj>qdw>)JiaCQ{2wmOg- z>wE(;eX5M{e3h-c=J`iX+N?$9#hn0>d;6k+r|yVBUHS!pvExY+b`!-wZ*D4tk<7=* ze%{e;NPF8SLY%((C5DF>%5n~T4dJq#rmMN9Y z+V&MLT8G=YN?`Pw@`mF%NWwt_!eZQ?-Z+1W$@a_j4u}xQ5w;Wg-G6(5sSWQ&IeI&*pD#*?eCkk1hqq9(hC|kg z`w+CSvmLH1u@LS@A^N!>I_5cayNXGFONC4mI-+>0K@zqFn`k zn23RfESxNiuOxX&7umo;#?C2r!xgG!rKxDwG=?KXlxxsL?+3oPdn&ZChvhjRN`DJI zE@~IwKI!h;ej&X69o}$I|0XS4S*M#_;_kwuRldCP>seW60~;1gR78B9CNH~LUoL|1 z!b;lr9i_`YRX(KcI@6ZH-{?yJZwkeyMQQt3Jyg@?jVlu1Zm4Wf&d0~D!OB%^a+G{M zdV{=SS*O?3QPsL>GmAF$Q#?y|R--XrpLH0`V1!c+NEk*t8GuQbnyg20cQ9=+B!?Zl z!A|1de37}Z{$+uJtVxpnBlOuz4s7_JJdBTkoKTy#DQ>5&LDCjyEsV`ODdlJ__3-gl4{d|ZJ> zdN^TAz}m_-^I&*s(f0K#kJcq${qIlvNiSa07{~LnU{4!`%_%y1>Zfy72iz)+%@qk% zOM=ICH1-YaYX^euhP7-tk@(!lU*k5~tjr*pX~7RdW;Te7|6WT^2sdY!a7`@X8=t@M zvx0(+&W(%b#kdYzQypeZ*QnxKqu<+e9bCfN*)<=RUO84{0;lmo)x&O13#oUzzmpPI zQ$YQenZ>smdf+D3*tc)?>&cF&&iV)0sK8!i%|%ltPsY$vg{pwboh$xSbXukMv~C_d z+m7Qcn@b&xO3S487xy3mu^vs!WIb1z-yZV1DK$2F?5_IYpyUK=QrM=O z-XHSRbw~O4Tvn|`?YMT9W|e4uNSErLSU8b)0|gs&c2-*4K)X0d%YLa+Zky(qJ2(He zaWlI?1xJ!G&t^y{HqmRtkP$jAHFr2f)IwrxFlNe4EELnqPArVFP)R2>vQkNF*UA3x zDV<=9M!IZ}(Pi^wq4H};zjrP4my!4_V91g!ldNRFtv6FJj9ob`^YbBO=S6n4MDRub ztF!O-F=}1k?|pILWAE~!?>6vPF6dVe>^BGEw`QpCGN=xrwJPdq3wCQVBTgnvsjA*KTRsW^F`i&*U^V*8_b%tg1~UqI#Devz|nn#oh1VM`kT~*uFP4O z*e>&&O{M8aL}99UnT$=He#@Y6UO&zfG8kw}G|=<=gJ01)Jv8b4>?$9L+gTWo7>~1? z&UdY8{CR!peP`YsdOX(b2fTg$bsIa^)R&wjI4CW>i>=e?7=}?GO=^?29(8FoF^?6U z1#4>jVglvTxh$ISE+*dE9I7S}s1>!Iim8Kl)t#62Wa&L&-Mr&MsdR;R}X}Z z;pka#Bb^^bG|Vzi=x@flX&pO|Tt_)!wP%TXip}`~UJCX3?PpzkJ9#coV?LruDIB!x zs--hRf7D=w;k1+2!B`6B1BJfnGE!`}sXBxZr+>D;>%K4jY--HhHA7V^CT-LLby( zNM6k_npBpPI`ux4+#tc+5;z6wr8{7zomAuXghJv++uVX7>Fa@{crj|{w(u}2y1nep z&Sl71wIyvl2T&0zTbY7dGy6aG?J|ljbZ<1rm#+2dOfpJbC)L=`ltXevAc{Ovx)o?C zVb(Bp5_u|YKft|>AqLtp8MXGIMG-B|OL1_A@i6uKYk*`!7#P>)Ejj9V{d0LMBUri{ zgB}N6RLz1}KSMWD7%WL&^J}q@gSKm5gS)9%41=lr}pl?uAQpR!>5O~=~w_u4Wu z$m9WR+O?om4KJQSh9np2%iY7*m{R$GoO<#vg9H8AkFg zT{x88n%1pXI@YynVE5so8nP+UDiH3f3v4a4co{&Qlp9=e-t)g~+K-+TDlFW%Vu8D6 zOh%L7A;h;tEwE~-`8D4$kN-4ab*efGEIVcEO}S3T*Ja>AX@_e!FZ}(d=ARjM|6^aQ zit*~=cKSW4>iob%z2WQDs~n{N@dGY|>%8AzF;tD#)1tCPMySy^6NxsNQK;$Ux%&B4 zvS<&VGsICQn%1#E4|erHWrx&qFW9QG2|M28cAz8m7;3fmGi#7+CcciRdug^OUaz6` zEXG&u3!?57m;OL1!QU^TLKTZIe@|!AP&_NaLx?ViAyxwem_OS*Di2A7y1WT|3Nqbr z+KxPR6!+}p^AuhUHq}yFNM2i#J`w2KvRhrKZbikQ?i_q;LyI2TSlhY^>ftIc;E;j% zr`2UX5X#>s`#N-rQWE1YX-BpdGq$&I?yCNc?IhsXMhPy}6>n5p%!-J41i=qJHT2Gx82vBc~lYltI$$SASXW#(`dKq0#s}$$d#PD5?VR^eFwG= z9(tz9x5t|hNo!uZUQiMuaB)#QE`?t{@$`FACrQ9|DOUtc`X$%|Kj!n7?bFa~>58;o zQ{YVr1UB52iv{&Ui7@4n;p+t9{#ELd)oWVR;}_m#dF)_gga#9?i);(0ERZwCLtFf! z47Fu#zH)@Pq@Ei^ZHhcHQsqS3s$P89yzwHvim$z4ZH+A3b?=O%zCn6mvwuD=CI(53 z!*z4@I6uI=Zs1_T#UEDLJ(-$|$8+sI}d|G|cM`V@Um zORXNy8h~$SV^WU1y~TBsACjh%gSd<<+n~^dyw5Z`w2Nw%f<; zd^gCp)#%z1dQG0s2j-XD8sBX1_ELD!c5fGLVFc@&L-vP+e69J{E{+khG42Eu^7_R9 z*E2cVOC<#QO-!i8+^V2=4_YWwAm&d>6bvHY*ldp6YJ?{y6h}4%m;QYIkwsf=J5{xt44DZc z-G=$l(IGbNJ7n;ECYVqEw!tm|$h$zVgv@H}8XZb)$R0LJa`7^BVztaSaXYy8d%jL9 zhXQx0>al&&L=FV@Ws97Qdc4Edbay`yp%F>=Sw*3n5UyrB`& zSyGPB1{V=a+)0O@@_c_0=G3 z*}CXJqOQc{Zf^&tWh`_tyi}oP`T8!DY){!+tZrR#_c@k5e_?If(0<1D0`NX@l@@tt zYF(|lw!SF4hK5oNuEF&mM!c-G9qc(S&KhwGsZEM`{kcj9Y`iK*LtjInbe|>&pe!%( z&7mUp6BSpQl)U#W;--+$=UWb~Or<9ZNF4+KQ`LI?D zMrZ*{0WQ4jG+fEeG9*xtXX5iKOysfvH z$&jB~3Xpp@LQM1LycTi9FaOI<4_EBw2FACaA2g~{r35#N@Z`=qk2VDlRcInIzBY56 z#HNdFGETsDI#%#@e|rEq8z0~9H~Pw-xw8!4{8K@t8Rtr)gsV}U%N}1VCwXiS*fh%Z zjna*KaTtAvT+&9=&iA=t(D1s}1rfSSJ1%K!mw0UK8(GLRnIOmnu9v8cFkP;4glEil z4~iDjxgKhrZf=KpwCUCFyN&9EnT$oE0x)*0TY!Qndh?wgvGXNlW)t9^aOfu+ z{aox$3>Y4@*)UdqA_c6*fh{r|RZFgV&q9JyzT9#__knp4*O<{ZO)e@Rl%LyBSX!2a z*t-u=<)8A;r+Khzxs)-y&=sQlh7^IZ2kvH{x%oC-Kqw~kyv7$ zlo~C7HmS|Dn1nNx!-=|>II@hk08B^rljf0~gJwIuf5Ne;6#;B@NBhkCYmDDK6C3&|&d=Lf%4dcZug&6K?7c5B<#$}hVp$I; z^MU^dSkU~pUrhG#u42CNvmS3XU^lRRtgg?pBVZaSPHHl6=EFN=^(_^lI7qgwPdn@b zr*g1FL-whLHl&x{uC6>++}wc;yX=ldiJGMtJSQ#DqGEETGYD)FQ&sjTO62WQ(8Oz` zP3oVI&E^}hJ#ww$;wa9knmZU)4UKITG~D6kjs@|prXZiEb!ePBF;c-kanavC!}mc- z&lm|y6L($_FSVI^7?&3uh~v36@4kujreuyHDtOI@$`XZbPas{#4uf*-J)*X{jyJ5# zEkjPN379z!p2@iX9{>qJ_P^c)_-958IkJfwIipn`lS_;Uz>BccsPq|UT(r87b19!1 zo#ZXcdj`1OZgy`biPIev|CxQubKX%Fmx|?g)Am1)BO<%Q_j**B_#@KAng#B2*-bBBy$}?R18=<5@gDUC3R0tFX2LuWiiH^^AdvGhO0rO0tMQNU%E!N zXP97W0d@D5Y3h_qLGFI9$cxEWFW`Xw<=p)0#pK2B+M0PI3zEqP<}K=Kgh(QWB18HG zHu&PdfGc~@W+v!u=E{2ig$Y*hWMy__%BYcE$c*Y%`mo{Y^t&h8>c>Bri2jk;ZuoGYUNObjVR zV&qp-VKnn?CX+jsB8GMk7x#U`0)T1S86Bn;*A1XO9U3CO)hU5fp&JPILfXU@)54sh zpu9+-QCnHQ_l8x_{5B-n%v!?F_6PL#PRKi>L1$TKv2H^FtOj7SPXtStQnXllcX3Xa zb|TX`{%s-em>@}3D$R=SLs>8WUoe`E5f2dJ;4Q$W^4CA7 zDIAM$Ja6clv-Dd)rkeXW933JEYS|)PAYp~}(T=evS)%wlCfeA{RlV?!{U<+;1uuXZW zJMjOB@g?X2>glJVquqYN$O`nTGX;taUTDx`Y#tm^dq?rEjO?T|6Z(frYCuCZ6Pg#5 zjjofb5P&{{-6;iLf%-n#+-s&Q;8x4QdfU}jhOwBiKG8`LYPFNbN%XXe`>v$rRE86G zZ(ZCqfu<8frryf5|J9)zxzc>)NI}qd1)46tKyr8^z{a)2bWZPkri#yo0`9_Zh7)6d zg994FfW}RAH0n=;dla8YCu#U{pKOHBMh;W=7%GXP>OOYAYQ0L%45f0s=1c2Yh*`xi zYI2qpwFOKkWt7iV+AafgW%Qk(7y&g4UD$xjv~Ovdup5lB-n!&01?wyG^Knt_ZV{(e>7~& zv~Vt#xJ0jlV-P5-pd1+m*GK@tQSl>LVWnCIZ@p*JnCTlXS^Frib>zjh4<}VtjTPgg zveQ;}zt%`SOM9ii$}QJ2#{OR8xd1HHf_Fd&uRg*Fu_F*dssnmuAk~}xJbQ|)Z?ZU%(Q^xitY7H9RO<1i*&r8fSRyeFg2=;$(eZL z8Xv?o2*5jSuzj!;&OmLw8}+@2T#0JO_A$FYERE_}N7Y!#RD5=`>kDA2o-yol#A`ja zk8qH^GCn>-0#R0<8g1LJtYL_!yXUy=Rxkh|-kR(rS;$41IpAntkwMR)r4gpb``EsS z0oW#>z!Au{Hc-#1{XijtaRiN5B!`@cR%ER30YZhXfgi6$t+1~Q;M$?NARwJiVBmhEMuwb-ENo!xhY)=+JEW@azQ6>#Vo{X z{RA6ciNIr}NHFX!2=reQa=|Qsg$!Z@v?(T}2P(cl7$Zd3YCulFqL09OZfSD&K#R0b zH6?Iv@&(m4Q(=TsPAHOyV_nZ?A1rW%b2$l?W=#+~^#Yi~MQD!}T7A&9NJ}kqs(Au- zGCn0zIHPushIezfoCguPGe`w4UN`nyGz)DTEl;o_cD3u1cT)>wljyPDpvM%FTnfer z&yD&(g(fMVTGpj}-Po?ZiPOY>oCB0&)ef2#vkagk9%O}Pgq1zy)MX-#>fJ@LQ`@c~ zqh<#K3OF*i?SW;P`EM5;?1_VKofn@8SujZWch)x>A$pP)a^Zay^c|mYVac`5l zd;*3@3Aox2tyK_K6tY03v^tD_YZpF)^p&3N#khs$?R4+hzebIX3*6ZX5h3EUlRq;} zPs#TRH^9?v?>>#vLeafo715|}&$Gom=r`2rZ~$_;vLdw+T@+F&k20~-!5uWdgT~kL z95lY?I6EK#`@DbeLfNHGtJYR!=zPOCb`T|Lt9^?CIeVbXi9l#9E@sXw``-2Xm%<1u zRu>3vj=QXOcsl3>VG*HpAPs(htgM&U{5?t7LWd0`Y(lOXO4BQyr!px4G1)twQl=@- zqWBRBn8QsAJK7`hC%;0YgFB#maZdTh!x-N%gt2;IbI;V}l)Pt<>5rTJ`qP36qmI6z zJTqJEw!E^2L+*Ip+Z#Ab{GAQ6kD@egs-aB`MALIBjPQBSWvN`SW@+jm%aqx$yze(qS1!g$m;GCm}AZ|&ReFz5Oc z8}!`#RWuhP&oel+{%jPgy83L!fWO69W&HTok`fnKT?K+CGGCOM1tiB_{aD5`@RrmN zCe9F4W9&O0S25LR1Ncs)9T_>axH3)8dwV)oFOB8OU6*EZMT74qXg;t+I1l`VWljAll#I}023bu7myxk3-F7%@HcYDx`R-b{I+wg_k zddYgxkXD~<^u@uaX)`kPXlP379X}Gf((3cLXiKZlzBG>Qg5s>H@*<s{XF(2NYHO;&8voGrL6GvuRs1&8 zF4u=0cn1welV`OWJKSUR~bt2q8liaVb@zTp{DOQkUkmgh)+FxYpcP%ly= z05#nGT5iBzs}G%=;$qIjeEleI_d1D#P-%buBG)x~)=^Spzv?%J-Dv%DBp7Y$3(7ri}DehUJArxdL_|Lu{U*QkT)f!x>klYHQ)u^Y=o)M2N ztV=0^w)+j84NPsuwF@yzWS*CT>*W~;8-}~cLY<}To@Hl>&&IS!=3KMHFq$*E;A5De z0ITaHPa`5nBIrbSux9+0yXZ|?7hC<#y0GN;udi<^*$8&u2G>Q z=EFpa6(NyLaU;O1@)lg9&cokhtH1D<*y6YJl;FIeIm<3+SAo^Np9P5V9h=SE>NNC& zl+;edYYQXw)>X*#mJ(kxvtSv!N5q_t)uF<-^`7;mb;mi40uYpLe!q!jE}8)k6Uxp5 z;1=`)lon>EG{V;%TRLr!E z3l7EhauczbJD<(*dW-g4OQ7%AQUxwsFTjMhAX&_);dby|LN%Mog8$riuufDW7j~;b zh&inaL%pR-TffxFHwuHk0TC3~f2Di(Y4g|vpjGeusAT;X?8sBnpD{3O8_{TI9cHEA zKb5SiqVL1aC*(5F+v+$@UUC=_n(4VLXMI_rko{VPH09K0T@mV-EEtM_zhX`eCW=+! z99QLL%%mVZW%)vC(8=J;hCxC-T?Tcc zdfpr)A&?CYABXboQ_c$Y8`#l^bJ#;JA`WXeYF7Lq4HYnlsg%rVK@-git~5^&C)cS? z_S~Rq23Waf`{%MVH+pxL{Z->1DBFggo)xYd1+*~gEcdAz5r z;{j&0k%AeKei6zj%9Wd$2vjwck)`iJ8P z4aE@h%Mfu~`tgHe6TYKCBsR(2e{wXS?T~gbJdA(0Ml1o(7$e;pF2<(0HB%iRbB1cy=w5ZyusF40UJNX?h&AoySGmiM)Ip zBJ2!X-zP)Xc`8($r$y9x9%wqx0ZHe{P;~Z)pz}P?bDk7ACvKr08!_i8&~gS*`F@m~ zP00G*&~f&Oj5BE62C~CAG@J<0Z)+r+ouxg(rHxE{JbuL%1xpNbdASnDESTZDhHf9X zw)H%d2Wg^9nziN*RvUwWI&-=&>hv#_Ludp8I7kli0=Y-uo^PVB4pl#Y#0a8*hZihi z{GO=>Mh`^E%9kKi0;jD{%6gQdvE$IKq$|$ufp?Tm4CC5Gh+wEKeqx7VM4^mn8=Myp zC_u=ZiZnAaSlEr`4&1 z@GYgONNJIhZ~VD-{)MH`^wY#za#1_)qd5h0>8c;Z2mJ;4UW^w7oAOTxXrkcPAN1Ji z0i^GA0*7^>GFE-Xqd37HE=r|k7*v#(s?Ru8D%R$D^rM9(^y1uyBld}!FD8l1&+_Fs zfo{Eb@WasualW8MU(cAFb1Uz?xnqpb96p|?6n+S@fOx2$CTo<|jvrcbjyslq5FUBO z+?Bau+r37$rAxgus;XIR@j*1*b*rB7zRV&BpPkxc=cQ~s5o2=UZ|!?H+n3_1kh6j* zcVqyyEkck7Q3~%zNYRw01%N^+{2B#aNBjJcUL~ zm+KQSw6<@BiDpI41NmStQ5Y~6T+LZpTh}N_8?@$WSpdoQU&|>D zt&+9{V`;C6!X|ACnX_1%6?)XPZQX16&>VkF{7@TJl9*{20XjevrH+2+E@VHV+o0G|(-TGYY_F}E3uSYb~-o8`sPiUv`wa8#@}nxCzXac@Lokg8lb zs!}rHfYJ&ZQtk0gKkc*O0@-3tgT!BNX#?Y#nOA4jdmnv8quk_Zub!`2K}DI-uAN%D z?R!vwqe5MWg}Dw2aecPoty_h*_Ncas=z=D9s%FveRTy0>)&T#*g*Abi^Ufjk6Yugt zu{|iZ`&4YJY@CHG%z1?L_$Xx6j21JO)f3oP{pQ*PJ*(3&N4NWQ8YgOe5L{KDc_^Rz zQ}$CSH67U?$D*H%p+gimYyAS7H8h+Hd=9Z!m@BuCm(!+31F36x<1{n6fb9b04wx}X zXqJ>1sc1VMy1qgF72;TOg}GdiG}$-}BbIy+_NBg6fg5cR7KE&ZRc<g6xJRubOnn z%h_Y?G~kr5pUhum5hSHWP@xKtF(9_x;%T3=$BR}`;^{#SJ;Ir4I>l0Sut7YT?|lVa>G-`RX~*5G8s(C$tyN>#PBjkax9)Z1S*!pf zftDIgg|LM85qMtFQNS5h`l^t(?1snSTdYjA_O*xQF#mo44Ak2w+y~;|Nz*9oY6ALD zkyHJW7#1@&9Y%aJDBRi14DHNjNEtb|K+nA9Xjl63FNt-dKs`d|+i1b6CIvPBF7$}xG?{Z@8421B%qrSYO%T0iV3|dc*)$wA1^}#g z$MRUMt*D{qTwA?Ukt0WHk8EuZYlaRvTvWg9Y{3!!oo{AeXw`on*>hfbTJ0gS?6~Vqiu`H%ITs@ zwnLe#F@|;5NdtX-t;&I6vPN@|t&7eB(@1YdwvF*B1z2IwFadU%)z-1`T9|uVaCJ|I zkO~m1QFaCr?{*pkyAfF%65vJS;V>q4ym+Qv4wYB_+> zF7ri1DU-+A#dtA1mdVt;6?XY~eKV#`?P=JF2ohfH6}i+^98^c7Q$(Xx$-n(uLpv4LD+G~c@|I?Pu-P(z zrmL)YWjl+)0!R^bwI8XpShJvV=Am z2)jkq%Gq>Y%v!|MINjpB){=uyS4@MMuIh)-Vpbv;g7b&kDR{uLt}5r%*R)_{L6w4A zI_AEP_G)Xkz)4P*B%u&-qEveX4SKDybgUC*WPu?8Sk4Pw5H>i{c4lu#+rTdcNh|>d zpdFk7Y;498O-|TkHmUsy$80p2tY9=<2HDIleN;ddTCr>j)leXm14@8k#Ix{IavhDV z(1#q8Q}0e`=>^3VMB+jee@9s|Ih(a9Xq}Fh+VMJf3$Q|XS58;ffxQ%?oAn`nuVa1N zaa8LDTXuv*WRh|PKIY}PV40-p7^=A6)9k2=x8_`Y%fw92Uy`oe!6*g{R55x=ewl5N zd6>-CU#bbYKn{qB|9!y&W*Nsw)BUx?vpgQVl6_jGw-X`9nN04cJY!u|_9AB5Cx~^T zO-D*8ZPvkCs@P@T>iX!DN>&Au=0x0{HMlkFc+G!q9RmO-yd+;WtHE)!{LiM7SB~S@ zI1_7&fVlE0xw(GzXB><&H!dv`Ofj#-hAt}Z@Eu&q5AqdMpu*bA7%`)C!Dt?eAr9lS zS?Vy)kLT~-UB3IvOL8%%V&)QGV3Z{F(?K9lnrMd&0leH6xe+`OU`mxAUvVxVK7cvC@|STXip;Z9BT_}N$wb1 zpj8c4vZZwo7?GEm=8FtA@WoeOg0oT?p{XF56f>juJE^E3MVT?;=vY^5l8s3k83IDS zfQJ8@ylBBb3nE3n{KW-$oGzJFp(}K7hQDs677XT8afFzIi}=eR;|ECFboyV zYMWcmTPZWv7y)Y5^O_a+thmV(p2%(1^unGjt6trFdu>EbnlsIcmAON(b`gls4vvhS zUtPAZ2iP6Sg|l3t+G{@jn9w&oV;@K4W5Nn;|7|G#$2+!!AH$k>UgKyv|GkoKLL-qw zE=)U9Fb3Q5=3Fl^iWzS2JuCPW;fX$XgRIl7i%_A#L9`zfgH!C;fXpk$}6BFfe zQ_GP=lEd#=w)EP;2=cY94CZS^)~i9rRGK3pKw5u=D8@q}`#t5bAm6?E?xoR@ZJc$f*tE&Vg%)SMx^BF zpCZ~F2$W+q;Cs#<&TQSLv00F@HL#sQBG1`xVp5{2IlU_6f_yZc-j+N|S@F^y$&W~v zsu=k3j0r$w5yu)FLPqX(qn>d+FK;1dD2TmKe5THVw$IQqboTY{|LZ??dfA+@ZXgA? zwl>g*IkR=#SU}y-)Dy9icdPAywXUWwQT_>ss-*^8grYtE~iwx_Y`trjEJzV7Z z@CjG?A4l!E21Jui2zlF|jz_?S=6$WUr90MJob|wY4xHz}d4A=b$C5*2JCShh|MF}n zqIpyaBlWZ^s^`wpb#V?i*2SpQZBFR>>aw*DWMdJB&Wt8j-^>8yC(sqmfNbmnUa(X` zQX;%KHaaO28lwKYf(;L9W#2pwYGvDVVD766a$xTND$M<$Rvy&KgIc-z?t$|hIM0Fe zG&~2j^7*Tk*1x!cVwqIR8JTj30*Uk46V$D2+~qD+OD+^k%7R_r@x>3|i;SGtE3P0h z8V6BRIDsAY=#w!g$9irzUudjO_AJ)*e>L&J{2xd4KWIAp zh#`65Z`GWBY-!q(_SS{BJj;k+2~&y|OH4ykWzmHt6*zppbICt< zLOv*)J!IR5Xnj1AK7-r^*Rr(ZyiAEe4B`5)VRt?fA&a&Jh1f+)_}Y@Yby(&ba)?UB zEHJEQN>>>G>(ik3lxHB-(>6+0ft5M5xT-U#lbA88+TtpENJ}<|YOE!Kd|U|+wufzn z946r_i!jDKyiFD;=N;&GaYQ@JQ-z{4 zz^=^2D**QuQ*JHhwFQN!FYU1k0;-!kF?t#h*<+jMQUgRwBuM@myXAtse{=Em*I)ma zJG@Zatpf+oCk7QPqefpb#MdRxo9{**eSG`^l0mK15@PHK z@iv5^_JhZ;9Xx{FV7Sd-*Y<*^uoXO8J3+*y=(i7SXdCFh+I@C`ew#qIJ)qqd&}#=6 zY6EB?#E-<2z`*>h7*jyXnt4&ff^8%S05U{;Zf5IhzYp4Rm~(k8k<)>>te7@HZ-W%i zcBFtnOEnzO$cUd;HfPaY_^B1PrK)mZG1U<83nXF(=GS_H`jwA}?Sy)@6g20FE246a zRaJnIYdiTqT=%N(y#hpW zCUmg`cqf?}RyL`<-5G8a8*CI4h%+>_2Bb+~sue_pgOICDYa!D~!|5Y(+S`XSvOa*A zSFghe6@$zc=0MPGaT#0V+MA<#yWmg=%9d{FggWwrigR+rge`O;qmWn!K@5KKCfLb6 zqZ^KyjC8iW!=YtXdP}Utmsg1{t@13(s%%_RCH~TXi#!rPxMg*TgyymoU}rL_U#UY* zYc;&CTQyKWstKECITduqe6=|>uE_V)J^IRKx?I3Xz$4}$l8mTrFA)IRQIP8$kYctB9BCaLgF)hfD~1(F+DbJdAY(N~ zi`dLQB}TKyVSHvcvgZXgZ^6|V3IRqcP*Rfe8EcIw*b11G^a~P=FsmEQs0U@0bRezn zdQJsRxR~wMLT(#fHA1v_joLJYkF~KO?n}SjdRya#?X=%`f$G#3?9+l394>t3RG=?_ z&H$JuR`#<1^5r6tIifFvYqh0$^ESCH7`-#^KF}6}_Q4u|u*M&(@ds;s!*j64cU$Av zILn?nUl89N&Hj2V+67P1sQWy>IZ z90(lKBl31-!0)7B-UZRwPF-lfWty5h_G0ql%ZUEpv!pB(zi00mO~12+;muVgulg03+*BpKcCuc1o+iM!8HqPc_&kvsnyNtTxgI3{l38G{j1QQ&a!ggcC{Ys zC)v$Xc|D<8D|`Pd8aSsS?JFJS$DS>_y}{OaTiqyFyiw>~v^g0$qXKJmr2Y~G`!lc` z+&982BhH0}0d26L~LaRMV~?5iRkE4^*j)5x4|4h>ZGSZrW% zMTiT-%X3x@tFxokA$#(+Hs0nj`5|@o(m&i$XmOu2vXEAovIP(!c!s4NWeV_FtIgtM zauCH_(LYjEhK@L0J$q}&Nk3P>&pNvT#&IHxmGeI$tqfw|&{s4GJW57c(AEeoh*9P5 zX~xiL<1WkAgkP1n`xNr5GzX$fCFA)+cC6avy=dBhbVN?Dm19K;DfRXY+OEp$Pm5w# zEA#r(f`a0)c%&uzGn>&uEcB{?;{44D6&#-19;TTXLu1plJr7IsfD2hx7OU zcz6Ex^(ly=%*_CbC9jP#Xl`G7N8I?$#sDXA^MPE3QkOK#mV|vuveMAQEzd!JSs$+? zwZ5^h!arTr31wz$>JBOBb|q7F9DZvUNWkh>8){Wm&BYjZ7cd4Nw~${jAlWoDD#Iqwj5zKDRdY7F*v)>2I$t#aPkNDuRk^&oIa%) zdx%o^J4nTAn+6*c!DV*lHbd5xB$|-d;CL`XnzFh9&9d6d9WHKAumYarJRh)JsC&MX z*JdY5BPCq{VW@rsx9^>N{fQ-|ZoO!|m#o8M7Q15Ku%(L+#;F64qzewN4s=lBv3u$6 z*s^2YF45X{hrNRDXH*>_Cse%t#FbY6(*=#@w_I4M1G@BOrA)7G#YGA~c6Hyoo^Bx9 zE~?Mn4QjViGqhu6Q_`eRbG~q(F|GA4@xiP#H#j_9xI{4R*FTjso75Ns{;K_&;KybM zD!E8YBV48lEie$p){#@v2J-^RNjlU&M{labpis+R&_eU1%xFQ(t(?iC)y>)u0pL&M zHA|#O<1B<7dv&wg;A+9$1uM8reG=r#wXjx>K7W*)Is)WxeO)!Eg&5C-ELeIPP;h@g zC*;pdCo;DO`f#uOfdR^n(O#acGUnrzoyvmUvx1zY5^JO1^F*Ic$p6a<*{PWbHpBEv zdkLL#63E_Pk`ri-@H}TJmcu@E>XKS2&01>}ch_UZr$6AWvx;KuA^E`-wW}5lJ=QCy z!E#nZB>PBURoqkA`#7p-K_IO<8tKoINPK&lH-%Flrli2pw z)%M33%)wsCMOsOq3z>FE97Zw^#9kz|FiaaQBnx1uiM6?ES@{z%DZ!WFHXXH-T-Y#P zkVF;*Qwu4ux|loQsmv*Fg{6z~DVgyb-Hj>Lte&(i@#=@5C_w6unN@u)ddzpA(m_>N zbSa0wfn_{@_o`d9?#+A|no4rMx|_k6aHjq*a15;0ZE*+G2qm+TD}n6kvU$0uYmntE zJu@P$X#!h)I*{MHN}$JesR{H^+l)^j@b|tw2xqig#%`Hb6j-kuAa_$RUCL?HhT+pB zhhK^jc_+2`Pau-Nl1#mm8h*3W)x*K=W9_U6XVdNs(>IHQgK}%sB)eP+A`80&5#4wZ zN7+uC&L^Z8yNJOy3CZ*nF6~ww+Rng1h<|}%yo-z`3@|{zcVkw-=q#Op;qJ*F@Lb;o{}J70^$-i@35hSLTv3L;5&md?Pep6LA@ z6e3ZL6mEl;7)fRly zk!%@zHs01+bJCHv!fhoms34f$Dr*YEIKh0>O~h6p+v?DUFn5V-TTnpBQe` z9x3n2h1)q7rPCp1+sI5mUeKHEQ5vK6jK8lo7K6=ex=Du>E1dYOkR{w4X84WMHiD(- zoN{X?F1j|H4X%%(xi^naWpT^X6pN9=0L;a_m3qFqk>>o@w59f=yz-^Z?6RI*h>l!U zyTmb=k=mPthCswjrFq#8b#;nvW6Vmmwsy;$0n895QD&K8`nn9Bu4H;sQ^#u<0xszRJ?7-Z{o~e~>sp<2J(55Kf9Pj? z8!)|`e!9M658KlmXl5moGG()ZVLf@cMsTfiy2Ky>cF)ZArrgZ>w(RuQf-B~uIl12p zMKxDb@WuZ3EBQ`p=WzMx!vcb|+1TOABv3O#xFc_Y>AmXj zxYI0?w={D}r0p}hS$o%y-2T`x0ARa(1`!-+b7(h87vyJFNX#bD3ozI_UB%$5B_^F_u+7$*Pq_-82?NwQLDR*d|ys6tO(ARmW`0KwLl9=7&UHC31DGjudWW{K z^F)_4GlM$$di>&)NKt#ewVQh`4MTJ@I(g9%sIWTquNtOan>CD@iX{wXIGu1L;#@2r z=+YAkC!fstJtK2Io3kRwY3ULl?7im>RRlX*65hc=FZHnkqduxtHa*n&^sH0BoCCNb zUpntj4>&^LLHYWtjXfy2zwcYK0^d^LqNfd1ZIuA^>}n770%GC`r6m z%fgjRi4y~el>@-vrN~j8j2wag%PS`~Zr=)rHdDhL@&zl*W_9tobCFirD;v(1{Z!Hn z%qTN((G~IY28wCZG;5gSQUp|_WB7Q!RhfZ9vVR_fcg9hKf;@m2>V77<1oR)%j3q1R z6p&VN=sjeY)9MBao{e1emUYIETN^A!!D$tqa=YZf-ufK;LF|aVj;g**9YOza4XQCp zjpO`rmD+jhS4}$Vl9e~%Nw}o!-Vn`@55B6;2_N zVvR!7164aV*vTx4>k_EPhzRyztAs{Q}*xgqDgqOp}x-v#n08{rco< zGA3W1njjeKv`z2nMqjm@vf2m9q`LZ<9|I^}}v<(15MvUL#E zyR{Yr{x+8nOb))}H{e#KJz;#3wv z2zjV;w031AAsPe#jcVsK_9D0iia7fFIOaH%&;Fdd6Z6gXL9sVgQ_k{Y;bvgF(26;M z*;yUW$bW=d1G;6%yu0?L5rue)74WlA_%nhfPg&uCR4lmz_U>lcAqrM45JrrEk{#lZ_*Mw!4^A5g@_!4T44!?62yjb z3lxJvqx};-&y*Y)dgB^ivwUF}_TT7lvRsZaZIh(}M^Fc%=(@}ROSei}+Xu@gGjMFL z9Ic}|bd0N+bu_|NH75-3oSlTU&~Fi|BT`7LTkX()S?}2hYOdFxZGQ3wy*Ar1Hoeko z#+q#}_PRb)XI>EeMePm7h-XX$bXYppmeZjnv-MEMEYORsv zwXF2oE`!pT)a_c04Tg=m>_m%z;&76?LM(1eO=kSwFv&FoS(70Qjfr!XEl9!AGU=*> zWggX`3J`K0YTyVi?a1a)ZI6!-S5d^VbUuZf?UF#*Ju8-E;c_B@{!^v+3^Tu`E~*R^ zF_@p#Wm-EpvP#YoSWaXVO@CT*`qyVKy3FaDXxmrtKW_)`wTE9$o9=Fv{T$w+7*lzd z{T5~c7VWQ3155hXXJ4JYI5BJS)u~yPz!v^hU<>zO+L3nPdIi>S2%|!X30b5z6x?nu z%mAy&8DSuIOd9dp{B3Sm8^Ia|Hhiwh7r89h7eO~>g}yNh*mx{zD|CS+q@81h zeOkyudmJ*RCgj@Z)1x@ta@35F48`?65?NsJy^q7oxtRu9{$yHg)-k#dc_MQc;1shS z)!P^bWr(g{fuJquw$-cx$XmkR3 z06B)&K0{(r$a|i)(rhkVS-3wK@%fK0|6o?3F$ooXS>G}-)AN_)#U>M+Gn%rtT#+H6 zf4#DAZLz-bp-m9AXZ1q3NmRK(8M#Czyd&@k#DLlRs|#~)pvbI4DeuBGYmI1g75pQ1 z`*KRs8bXP!C-2LIQi5$6A}LtXUc$c*ibfeJiHq5rf+p>#Qyp09F(~R0_~|P(^<6Sy zA#bULQl;Qq7I^5rnH|R8&Xp~uEumzZNs3Un z`r0TiOgmGAuc4)$%fj3ZAZ;)ltSD{2Bpmwbb=+FQC9kg9ayzHy<) z0KrK$b?a-xK08|^{dLmgXF)bfA#(8AJ`{GRDJ(I?gyO)m8wVei>@pNQ+dYo~qV1kX z0Mo9z8VaiI7i7oa+U|J@knQ@Mo*~$_dj@6O^?`4@rx%2~qHc)89kG%DQg<&NcYShw zP=jQLIY^PoMVn-A;Dgl#1~3wfcizoKo)4dhD8mqOSU{+aFUR1X1Lrw#o&)FkRdb%T z@(&S9`7h6QBAQQVdoc;Jx-kG0Xd^~0&f&(oz=Qw?#X}l)%xEYZi#T*%kx_K2o+9XM?Lyv%*nZ9%k35#(U+dZI^VJ7b&o*PBg{0-t6w$l9ebRp(<|oeNyOPe zyosc?{bEh5NHa{F*+i6iEHUPhM3~LQmwkvXPbIcIYmp`L0>z0c8x}wACkrtmN^D)X zKGCFGEQu0HHW5cKBIsRD)%9R~zqhf+7*yoTL7Ee}LuS@F-}c2E`|#PdKXuisQR(d_ zq3IzOyDe5m3YJNlg4D8BQ`S|`RjlAlf#u+~Ija2R=K9s28vuT~8L2Gwchm2RlodBT zXFqb0$`4W%1Cbe`JC}Mar(-LGWOO zc2X34_XWYXC$Z z;qS2bk5Y_#|An|8rwI2>3vfSD@$Fp} z-rj%F?OsCNQ*3)vfodNWq&F+l9J>Ntg|(wAv<0(7?oZ=c!V=Lfg=)>-`Co)q1%yvwf+% zWMCD;THjE?u-V9NrkEs5DP)3FECVKJ@AX19UeMlZnGfo9MwRBtpEH@kH^u9DD}) zz!V;jGkc(ZLmw2w+1{Gjnm@P6WHGMhscMH?8MX~oYYyZPY;;hQ7hRav9A?iV?7&i% zAzbaIN!EmILpvPges@+1#^*HxWT}_rh<(!kel#LS zpQg(EN9d_KnvlzUk@19UBUOqiD=LIV{J_YXH9erKx5qmbb|t|hI+qV5mE?huMZxY3 zy9BA!8|8wf1r81%g#dv9z@@uJBz_D#I~pKa!MptdKuZ` zGl*8om2beDJKKXHz_BOojV#<}XDPcsQz?Bhg5fFa%B|0ZDsuGVsJr!RCnb%-c~@;r zW#KymNXIXaK*~40CRC7&(R=35Mhrb!VhlouOcMvKO`_nnNrX*>3_KmKJy&(RC@o;h zR3dL+)hmi=ZK1X5_V2C^9``oKx!tkx%U8QvMAsfzCk6khWaRPylR#|0m80Zf>D8Hhj`A+r-El2{4S=TbafS9ml~(j-LP3jPL?~ zPjb^57#+GxxV~pJCEtNr3jXnJRwm>=aFYpR)F6if?hG=j z&I#RS@|M`MzKdCW|Ne5&B))%t>E>YmDW_}0haq;(VMoK?mmvFQMY@a=5dvJTmG5^W znob@;D2-b7J1#n7%vmQ<*WpdDY&siCDLFEh;-ji8&=1E=1cF(`E84v^7V zfl!fG49%zMOIM7x_YlH~ZEWl<6?A5r0s>2N{1MK1e~Tw*a~A$H8#sXRp^E9#NzHDm zU>UY1NPCl))|^J<#Qg0HoKdHf%9#O?sg+38*}Ay|@3#=QV#J?*ZtZAt=L_GzzubzO zcZJulb93`{BR7XXT(jbyC)Qlrx;)nUwLgZwx3iXhu4m1qQVtRcs!tq41;r7H=G$<7 zqp7+g)SZ*WLt^cUdWAvuL#la6~AIi&vkT`h3? zD{hRN-tmMj=F_;chUV2b*L73v1zXI?n`O&Y@ce3EH zji_3zZ^d=(_h|jZyu&-TB<8Gu0rPvsiuzGwI)POuqmj* zejpW|4p8A)=OzvmYp=E*)=TN5GpMs#K!JW2k5LcD7!-1`x-a=s{gjQd z_wgdFHf9So>M>(0o`+fbi6bVq-Bj}AYRI!RdhL0wemF_=^rWYc)ZkG4(p>?U${~Ek7HdkRG7iDK{R;-^66WBtGw3ld5$V|`S37V|) zz})=img|bUDW=JS6*-66fe=og*t8swA&tCfn~Rh^1(At%HOk2?(+`XZ^5vIbehFg! zm;d>n|4C#)C`nnu^Ys_ut>3+9sZHjd|9$@UTL@!cXhm+Np4(o{$F+vli2SeX@7`4y z5cR);$;&TGcb}o%_{;zJ_kZtsn-@?T7qS|wjG2YY{t!|V3S81j<(S%K#J8R>X-IJ8 zv$?kQ-p%>Vc%qTpAvU{O!7qv}kl zu9EJ%4jS!s+6=ic(AbK4V#}M~)ru*)uqOz(Lt&>bC?!1@03JZ$ze*K=u}e2u6!M>V z2t0<1Q;(I^v##~s6Z^M)ZaOqqmM%`Bj-{0eqP#sNa+jx9Z^LudO9Bd)S& z%=uP$VoI^rGUibBv<`F`21;Qryq2W{S!On+&;jw=UGa^&OnH(cAkOk(H;_WrR@47`1&dQYn zhYNFzkWD*8eOgQ8qjoi(4#e5+vaT@y?HSuC8f@OfVjBh)1*e=_KKtD;K5H{%vz~Rw z=aJbfK2O19@o5J6v6nU!ZyXiv2GFRU{qjzH_QpZ+iRGjC#Bo!6`XUVQ8Gb=rECMco zC5%Y>rhZ|;#t9<0BLY$P!je|J8uV7ltkl@}kk3QP3+QjO$dyCi8Si7FFr7Ma$ z#GuI5E`s~OXCQ7|Z3Tqb z)6Q&9(3<1+A|RyItZ)%<1ypE8>n1W6Qh@LTlx-=?0)6%#i#L%)VMI0PlqyoNnbAKB zP*g48due{L^)=gz>(~2uI}(68+p$9PqLBAIbux2E{<$Ixssbbfs+3GPSo^Bo>Ffj5 zZ;-k@#$GK;I`lkCLKlmyGj=R$xrI#GRe76nHNW1qx3H;g3|U@_nQQYJ_IM+O;vnz_ zK&hbNQl)%}#NcX35D7A7@u4oR+j3Iyc~_0!;)57f!~FJpv8eq%p;^|WeAT5G#~V<4 z0V{@+ItVvs&8-Lo7^0LjLF}h)38Qf#70D&yJ_sU(TkVmNdE5s9#t%R_!dX7}TMz!$ zgTM7(&ELAW5S9GWfrH>dM( zMcWr1C)+ihh38n$=&==&#-nRTuYbVt!9=WvN}>Xour_-An#_z2Qw}d0WM@ z%iCR<+y}RQaIPPm>&?%v(z$+#-gCn-Gp+#jutroINND%s40UY2s>m5CIIH>e-GwP? ziz4m4teg=mmEsmMMR9URE>dW#M~iKk^Kq44dkwYUFhU>*Ls-ZjuK+$E)KDQLvOhH< z8NFqh8ri%EkW&{lL2a_+{k1Cz+&j-gJ7IV5iC-;f-3#$CZ?7U})+=_XWX^I5|9oQ> zuLT^n>EVhP$xSmvnqx(76rfQpvis=1546oa1Dw~yotiFG@nD2(;$P9x6Rjuh=?u^K?a8)gYIX~ zK?VWayRxAK>#+n8YZiJVT*$#Kb27P=_+2!C4JC+SrbRun>v%Fb#@Mb(i754S{CmPg zBGcG2WX4!14T4c+9t@)=ayMRxI*4c#Bg|lY{rq_VwRZTAZ#ENaq6l9l@OZRIyC@#L z`X$OW?(d3ZJm-L{lmgXYs>g?k&k24uNfzbEHZjRrE{o;JH)H--YuCPki1SHV6in!B z83;r>mWTYL`*3=yr+%j@*CtrMJ--DyYV%mT<0z}gdy&nzOx%-uS~S-AT7L$LKqZe`Bxz z9@)t0>7SYlA+A0(X^aa!mchUpJuVBK^k3 zr+o&|XT2i;N7pO9b+)`~r=FRlS-y}3EtVv;7O$nVzBZzQ?X4~W3@_zvgfR+sjHSHH zcLq8Jt%a-g02KplB?1?2V_$RSXJ~SVDv@8#K5JkN|NZF520-MD3U5V*Ko^;^W98q) zZljI5=q2TF|~J$z(z^f4pi!6CAA9?jFZD2hH&^J))EeCe%z})C^CctPv=+a0a5z)D0AE z_{8%vC#5nYl%oE9HNXc~kqH3dxPXspihMV1o-=aoSF5rLp|{^^qN)Vd#g4#=<8lf? z9bSS}zHrT;%P4qOzm2Y9h+oW7*f_+ezCX`Wvf5Cd>gR|q7IJHr=o+f?#IUqSJ&=;R z06%{}tqZudiyLq7u}m~sCKij1*9!h%cd0Fd&ovpF|8s^R=heGqQ;cE4Z$K^YP@>Sp zl`#!@ZOoTm!hot=-glRm{9akFEAODF^t6^Su24l5jHQ@NmsP_pO3^Xc+fx^93O867 zJ*6tc1%{}CnRN%yzCz|i3A#{oslh-(7c}9zGao@0G=X~V;K73twht#cwFwsCM%PmX z^}LYfY!2CP_*Rlpg~^HS4(P^qts1XLx)e0$iLbL#$a}81w2mU^uWO2wT=~)&=(W1Y zD9_1>VvM|Y=NtL@|-dD%PX5z4NO}71|^47TW3~@P|x_SqM++?{I6IhWiMp!FkWql4i(vy61Fp zeTlvC*<6nw7?^fZ5~zrj8zWx}cjgLMRA>23YhK0~KF}p#8r#s?TBM3AB!Nv&Wjo)= z9bdCYqidGuKmigeK%xaRyg4Fp)z_!kb%BD~F#dJq%Zlb$N`ISZ_512j-LQO-QN6Q3 zkskvyf8Y}~Gn`O!RQMYTce~nNDyDp$$E3_v0VTf-*5IeNnKn6T9%O9&>>$0OwYq zTDy@y%X5l4{UU{CZ7g%dty@bKFmsn8sbnkIST!NkVloOG#Qr8Lul^8y8C+VUaen;;t9{ra~@B>XF@zLhJ-SU_$eP1>h>Z-PpY<1zwK& zuHm-Mm{wJ6C~~}rmbI0DaAY7~xt+8|mBxwmwu|RLNz2&`|L)t!A>KvxZ#r`OSbxlK+()q`bSHZ;ZuPTAyV#tk)$ThT9S@|qRjy~P7l+fJMb zWMa47;qBF5Ki!~1kTaV#^r+S@ZM>+{V~p-lfG@&jAy@fDuw2#zvI6%61p(3f7#A$y ztY%)h670c?l2jo8w2SHhi=PYHZ0a-aivH+qYv*E#3-u~E5NOmQGcz2ccnNx&8R;Q7 zG=3~A>{rX+t@msiw*+AJ3qOl%ANhCf!$}pVYZZe?bY=J0Q&4NB9uK$DV+iBbILerQ zhef(T>@P&P#l5{5i6Cfhf_|L-IfEaZOyJ%Ax6E9?3h*s%sR=H3*hfS3VRg#Cfl_YP zN9gj`4X-)By2O#i_xlE}_R8Y6z91$V$o+8_ta8 z2B0FrNP^}o!bDC)Z_W)qK+v$gQ;aBYaUa~qp#hSL+lOX~fXF&ggNxKqabRa)z))q9 zKYA_&h6n|Qw>S}RiVlRpsNfF?aJeP&HKSuyjsXi#YFL%WDu_Ho z0iMAKrKC`x5yxS80vDTzY~g%Pf(2R=E$}HT&sNP{D^R+D-5^MG{pp7GQ{~lPD zng4cC$f&%R#<}sCz%+cWoCVP4)tsb-Tr8|rKR?}UZV=8bjl*uO@Oj|<#mclRPU4UoW2$@dDk!c%ph4;7{1 zS`Zghaw9uF&ldBb=TKkt$*1*osg3EPkV?ICpX_*e2VL@@OCEH|;Q3s2Nm5ajec#P@ zq102SXKU*-eBOk-liIrLK^Ud2_B9Gb?}08Sf@zBzV<7QX_r&-o+a|06}bJcKP{**s_q-gGqd^b5L8xrsO4Gr zTL(p|hOgj*D;zpYWd zD@}E&>cAcec%UEIBkA{b4}usb1H_X0;m~EZuj3Oz$%e_GWV3`&@?5h6>dy{xFUb}R z`&&#L!Ow3knQ?h8Rc=lq^F^syK#nX?h-Xs+?@kRD@GPf-&e+&@Kdw?-&IWO*Xqz_i z`R9j=bht?U&wG1XR!=lCo|3UL&Vmfp{EX?s8VX-(^ggu*AeO}4Vyc0{!O>gkeqB5 zBdR`A(8wkno%lW5Vi^VT+A|7`Av&>B!LGKwCIeVO4``)Mk&bsfA@d5UbZ$(ad~u@BI-}Jxep%8^?U_Y3!OFrQjn0% z6Nnm%3-Uu{$OFaTFfc*)7-KR5x0F#ObQrm#Wqpj0F^tefQ6Rkp&qYwQ-c#k~AThv1 zx|OA{N64WmZNCs^W-g`LcWn<}kVwR*W_~sqI(B}wfEjRB7q%MJviVj*EhK{0J0q6k znl~>=Ou>8i*my=tt{C0`rKx5ktTi~zf@%|tvVLpa;0Aq-i% zt#GZz6{Q#h31^YE$JJ=DW_N!xaoo3AWcuijUtt@-q|eVHb8GuRt`TQG7lr$VqSz`{jeP3xGc)-;(T z-KNCQp^OfXO5Z$eu zFd57cQi&3oFAAA+A7yM$MejA}xt&?rSu&=M@|=~z>|j-8sW5-u(FE zyZCZd$JIA&Tm2x5EPdc<_Y5JflAM@^PP>jogw=FVNzJ_gT@&)~z|;6u>^W%ASbc4s zDr(l2n8hVLWkMTzQYdl)Uvy&^yyl2l25m#tjr^923Kp;0SHnomeRT$P#WVsZLFT&D zbE0@&W||5nOVwSX`m@-+er}(!Pk8mVg=MW;stX2?$WuXMMoR`&Mq_&Br38;f#y+vc z0z)jaa>iQ`h4xVDzP*-xRjj*ItP_NhI6SX@jKHTWJ^y*iEH}+%=To*|B8BQ#Z>pn; zYqIt7+p0}ZUuE~$)sL#Nut43O%PdudebPmD0nIFoMKX{I_Z-HjT6CUKKmjj+bJifMOE9 z?9qQoj{3^`47Fap3pkz2E{dqaUxQp=P|=RxGk~0Qh>S)AOG~uvxg`I30)I@JZ(vie zf+a46(iC#QVdRFT86IRoJngAMFM!J=kLV~j94ldkIXKypJQOk}$Pg9a^E9qC|SCsbad%wyo`E*?W;@39CEgr7dC1aMdBo7u0JVLv@;ft5#wNPcjstQA^-59}iC%P^` z#;cc)Vx@ni<6xKZS-Aujw9ufV@9G5l1|5|Qej&C!f|pa5J5`U`;L0~pg+c%mialm; zd}$SOma_XZmC_d@7@kX^uFr)ka@1Gb{+j1JqeZr?mD8@;FbvUk2GN9H9+4ASK(7fE zBx4R1%o36%%I>Gn<}~lt#j{9vDZ5~60lTf$>v&bMAubGWnvm};_podkcib$1d*ivR zdep_~X zMkpIV1-TSlSJk^Q@hy|WMpl_tt@~ugwZ(A8PRyTOYpvZ?#R;IYtRMMSeiv75b!RG? zuZHU&oX~Mw{l1?K`fC5K?uIO|VxKLOKbag)fcYLzj-7;;`RH5A*QyR<>onC*--M9w zTz)d(wXOvD`uN?dU?#$e)z5H`WfJV9?YB&`(^eh1;0GV!5Ab>#dC@v96&N#n=DZ zVhV^#E81BFYg=PWIm|4}?aus|&fdR1fA#kDByZcd80y7Dx{zFSV{8neUcNzh;@mHS zLSGhOD_BP@7c!Hx;l-c_tkQ2Vett~H-9^PBwJbf~or0z3Zndf<@ydnnNo0IRtMaHK z>e^DFwE(W9t8iA;@LI4U=a@IPZ`pNktG}(->hET|U`ury4t6iqHRSKA<66XSylbaU za~B00`vNgjo4ULsph~5CMERb(kZ989j(As$D3=6yz z)=jo3_&v|q49cK#F;f)|n|1lMHVt`KtYR_VSMe4ccvY!!xFGMoyFs^a_I2T0I@$R*!p;-%A3MFcpXI$Kkd~u?wwCI_Pm_N8f zUoe_&#%y+aY$6$r-l2sZtlZ}2{?!p4G;~CBiJlmRXR%}vHjOPvKUaz3+tPh#q(<}w zxxBu(zI@J3rZ!#=li7)cp3LduywvkoTmgP)yl}PlWss|d^^slOGj5i-`@H)$F8+Fc zMQCXjPutRxTM91K9Ir96fA9?LcU#QQwXl!bJ!V+z!714OY)OdWN`FRPiERcee(*+gEY)`zJer&zQ$K*sOi%w_i&a>*WQiL)K&fx}k z>sy~y*EOUayIkR5@64QMoAcWjPVWl-M!w}rW5NQy3=y)J&159FE29JR#Xo487B7(- zTOCCf-7(Ev`9d62_{e}0ym4)%v$^AvZN%b&=%QO${T_`jTO*oYGhCuK)4v`JZ2Z z3oC7d+RSCPn&K`ap0_3%z56V-_qU4`PrD%R-dqoKvhT~|A*O9B^CT{UpngEj3S$-w zs!{fo(B0c=kZ0*EHp_xb^=#b4yI$^osk#^f5YU^DTLkz4=2jvNEDV1#oQWyrDB|aG zCbx2ujO`RRWkZW52)}I!YGb1oxX)n@22S=`26eknr3HWy2{cqg_GS5Y?tpGUVO<4f z1OT*ch&42@ORK4Ga20Q(ttNcqS`w<9SYU%-Bf&SGY_Z9c^MW7uVCvY7nWLZB_*1hn zNtsJ6#RfTnO&``N#TC49;T6LO3WS*#-fIh`(k&B~!ue_IZrUyG@BvF^A5OHm*vNE1 z7Or{Ot?T70E7rO+T4m1}i8yFv{jlv_8b~i+ZG1Onp&w*11h=aG$x3*f{o861eCrZ? zUf1s$uHQ<9*0yW-oNnRc-@d)E($5%<2rtC|s^mVfMj5kr6-}aGmJ4p%zUhn`&BnB* zkpBb#?>I8!-bQB`)-=pAvnx`BRTC|sP&>jsR{qQBLSadbR6d9YTBPLssx2BPD&qIZ-;DX1A?2|!PP5@2od*&LRTgL`6{LDVoeQv(CH zLwv&^Yv{-GjLz20^hPA(cZ^KcUkX_+s&X&|Q%3x^l^U?4>Xh`9HQ242KcXsn-9c*a ztSgyki!-_;6TI?f#X1^{@SWHCyz;H$HrQGs6@8(Z6f@Yh9zT!_qxo3s!`qgN9v@eH zWx@oLGj57f=|7hu%_30Vha=}-to4FEPiw{ovrCdNg}d}|0zme zz2Dp#DhMsL9791BmsBzNw||Rh8>%3Tppt-TlReUB`fz}zQtGQC6Gj8nPpe$`zg}L;K(Af-XR^1!2SL8I~KUcuM zb4IRSzXcyFNVZ0t(yTk@6(66?Y)(!k%}xk;t)vFLTGO&!eC5k>Kn&LSHj_y=u8ChF zTh~gY1U@2jMpI@BIvUYtv0&+hkPCCuraXaqqULWYOG_jCLfUvlS2uG;WRWr~7&8U; zyl*gIB9Q(AwSKJ;Mr~!Wp_AP*`}=0&ds` z2y5&2M>!&_lR0wRm9tFxC8y~^R^u)+Gx};gQ9J5b`wwl8a{2B7G zU2Pv0T~`tg?V?~Bp~H@F9zsGg)e$xbg?dVjAt9sYPh#hG&x)mOJL;3+HE=hry*#A) za&(iVqZ)BT{pA7e4lakcK`G8{O0zzbI?TJN#Jp)O<_)Sb@2MX1jw&*5QJQtLs?6Kg z2Y%Gb%#Tx>d5b;Rq(1ZZJMaWGnxD8z^V8L7-l9_ThP9fvIF6gwYu;JK=1pog53Sm~ zQQhW^DmQPopgUR04eB>1#P}a2p1Huw6yxxEut0wo^mh z$!f6W+%8?+1FH_MYm1Scfle9dpHLksRJh}IZr}x#t^&m#m(ei&qGThIN^z{M>182P z1ad1EF6!F{ZKVw>ebQ5Qs(NU_OgHqN@{HbQPDh70BQ&f*zPyWGYU(T|#lv=H6Fr0k z>HxqlVTaX*2Hn<;b&)-F9Sxw3RmusP6nPGwbfbB zj=lNO^V!R#sOn`8HNg(tvUr)aGj?{h_b6tF-sQJ+G0;)tzpEAap|`TsF4-IZ9NuxMid*Pv)KrWP9l6|=8F=VT;)EoC)Y2(JFh{& zf%9m%C?;j&bgXi=f|-F{y@E+7;2E6_kNF`!)!o z6l;Az;l}uWz%~Sfr_SsafF{g&0?<#UQTwjjJ-_s^W`bZV6f+_R3>V-m%siM~^T-iDtItzJ0IfckQ@GI0Akbj|mX~AH5WSo1Jv3Ju9}wEDz_ z60JV*$_75$T*46gwy0?r6o@FI?{phWdS*c@?b=w*Wc@hlPnYQ8? z(Q1&0-Ne-20ncq0P`d@OeB; z4}2aC*~4|W8NP>IoV}oY;PZ4?AK~1dE#e10k7tc_pZcOu4mz@?FGa4cEQsOmpx8M# zDne4mDaXUif`bU=Ha-Ivwts$gIndM+eA!}b3Euc@%`KN`MG3;L$g65v5n#gA6D@;o zuE?$p?Trz!s+f~#abu-CHfBOpLR)0H?eM8$S>s}y$%e`?wBG|Sw))fy)W3I9lQkxz zohp9R_(%ualfFbpBo#tx4D=bHtD+;jrs+XPO^`7zQp21uQrCsgchi_rn1%2 zUe(krBME}VBO4OM;U0%q;LVVR;@{)mJ<_sl79Qi5a^LYT+q>=Yc}x@8rlzrxmwxKz zu0fq2Ok)SrSpV}GnZ^R4!Lgy5-D|kvEJU4ihO!F1dICo0Vd8_ek~tHZEb9K3lHUga{V z-)t*}4kYK5sOoEJ*k$xJgsdD|<#=a2=Qa@B&^@LG<$e%yf(=ZPMLDvKO>&mYVma~+ znm^Whuy34#*qW3@!GzA1fdKmm@~dBx_-K^sso(pmo+(%my`(Z&%9{F;)@oE)F3fM4 zxF`3tP{S_g79wcM@3|VDBSFRTyFurjAk61R)VlulMq%Hzjl8z(8Qm4K)QeJE1~hmp z?ZtD|XZ(d5d97k_lST`95od3%o(0vK72+lN|4u&q_LuSLpH5Ex_T~7${{GvO4-@#u z7pH$Z{l)$8+tbsNlfQlQ_AfVAum8?ZfBBmz^E>?Mmy^G-*ME;}k z23mCSqZVVX{bNdXJ9cC+)aPc$eb!-fwwA#mX6}(==5XB)0dq0WAzDAl%>@lEVb8W z!V_{%3YPE%w=fhGe;?Ty98U`ay)=iGWm8XjJv?*~*cGhAvoe@i_HKrT@-YzA2d)@V zbNTObUNt zV+dx}Xpw}__S8LI?hs<1ibTLl9`M_^6oS)e%4DNY+Ojhk{QaQf%~GV*B+0c22rXN!gqJoP({z5mTbGSj zs>cIgs1@&0+yem(Z@x&6Fu892ha_ByG|f2pp9iF1BGTJ0__9?3DyMo^s;nI)G2E{> zd`YIf5^~ri=z0!+3;lX|Jd0l{Ny?}!$@2q}C3#hbB0MxK&~INZv3tU8&lNEtf_F^Q z(u9B4=BBx+)wFyCPxN8hG(L?)S@yJpTi`DWr69247P+K*d+_gykMGMc1`<=ZjZpaC z=8p?Bjn1L0K;IamdrSncykHQ^Yy@b`UZhypih<_#J9#Zj6!;T#JH4NNm#7l$_p~7E z4^NjErql~INn|NDI4Qcl_UbkmR$P#*O3S23XA^W`A20&eZ&A+FLfY_)P|dibX?($D z__Tfx%S)h7fZ%;rP(kZ25b-RdJMoxAuxr3~ zYYTI))va|WtCE`|;#6&|b52R^w-L#0#H@y>+FW;xw_akag;Cls+SnT)CTd5y24zjv z+2g;yw3IA!YsK&bY@N#5ZZ2m9uX^7ruEjWv#mBR1#5iuBaThe^l6(&Ub0-XJh2=Px zmty&3pJ-O3Ob&1U6YlTKe%lx0c>m(f(aGC4!_2jpca8b4Mft6#=$=>Pj%w7)09L@u zq_5=;z3i0Vk(iS^Mhd}hm0=0N7Lqim zBFszz{$0egLhvk+R<8J|?nvlqmt=IZu{8&c)fW`7IrX3VG3Zv^Z-1KdyFtkZm1KjY z-&E&PMl@xo?}KhmOx^Ys$;kY%!7$(BrnL<0^GxeM)vJ!=Ja+pNckQYp>^kFt z>DBA2i_ra|N!i!cSSPe(cQjmKHTCV(!pLbH3nrx%z-EDY$8mIiO8%ysT6+kE;H5V9 zPIyKWZ+Ox?J$UOuJE*1_7&4gix*cK~y5^z*z+MR!6LTrFyQKIF2EpSY#owY;(73d> zX{*$C32RzrOk>xa>gf#rD9YIOwT*@+Xbk!qtcE_ z`#P z@kG{p1@JVs9*bb?4(IQ+#^$c!8*jha?S+^K{c=WkY=>p(znP!}ICu(PRH+U{t|xk_ zz=UPI)B62unC^}I-A%3TW=in#dD?ctgWaxdbkkg*z5g(2+M7zt_TPiFQnp(r-odvo zU-pQuL;$y70)%Ykw3+ov1p`Ax=TO_55FKDv_IfzBo2$LG-0aX??6COvUQh5+aTb8% za{2%`E?Y-$**0d&c9B}PjL-5As4QEYRIqE~I7&uPc4D+Y5g!kN{74ck-&)#p`X zFgQ75MzTZ*E{s30ISd{8mQ1E46WVmwflL0KGh(G#C9uj$PH3)^oYlW={saus3cWF# z(%XbrJ$Z5?EsFnW9s^bDo>#R4qk{CwddB49ckwp=ZuTJDxWwGv><-wcM3#&5V?T+d zHrldXaVrZYWPbLhu(#9Kn@_aui`x`;x(JIqNt5XQ^kn(i z6aAc(C9;k?QGX}jQ`zHzdP{RU(N|ngN9W(aeWn%*T*IB7bc8C@B1ymC-TKH0b5pRC zs9kt+3)8Rxa0n~W`lvZOfG^JH(YACHiG}2o_7n+_r{p$^Z`K<~e`Lw5*Q=eezBdfu zTXg*3ZPR@tc}LS@o~aYLn~t{sygRn>P1@QqEm_?J?DzqlG8&VVPsr0>j?SJL89B;n zQQQx|`KF`QUyjaN%_+0X#2n7TFF-Xo7d&I-lvR@a*Z(Gm!^4B&L1&W})*sV&;`VU( z{P|G(f#{gucC2#WWxyR9_e|Ld+>y#w?0$`;2nyD$Gk0uET$qx6>80!s{T;ejWoMUu zyu3Uo1r;=7B@@~&|(6-|EYm;RH8N>78x-K{SrF+<}e?CJUZRAI4zV z4TOmT7_q1lNqMhN$0dBoy_XE2AZmtu^`e1&)4&|WdaTO~OqUP2WTg_l+OA&?h$!k$MVV> z46SsHp6k~EdB;a;U>o`NKPXAG)vpe3*z)JrcTO?Obac^X8{CzZ<9&N@I^VqA-}PrX;0jrF_b^bi z^Ul0ZAA)_ie`s$rhU6VDja)XrBvRkr&h9%vMqi$J1>7%Fma95iBet=TvpkGF+q}Iw zGnq@?iwxB$yEOq5`e3i)byymcXiD=5lY4{`!P8XPH&qeSa&~1}iexfj!uOhf$79b| z%BT{dxn2{FQ#9-|H4T@HeoC?`BYBmLH3Gg&vMMcU&UmHFzWR-h#*WBuOmH)88KX9w ztf3uQmx_+CgWK99k9ejga`uAmsp}5{%aUA;;59k$%&I#ste_KinUtyHR;D$hmpXZ= zY1FC5NvWRi?tjnzEK|9(ai>1p<=JT7zVj%D{nk^&ga^d7)eS@IJ?n>hHo1m$xjL7< zk3S*Up1V#aZ>bmEda--@85`@SR+**zuuk@VfcBD#3G0jD+-9io6QzdUSL^zD1 zY8iT}<)bB6rKNgEN~#B>qgtJcYFji^pOb=W$9?xpZMJaQsSe)#b~xwkbkhV<0W-M@ z#Z)-G)TTmogQ+D}rKbv^{ccac4$bmQk=>CK?Eu(XY5@H2P&*t077Mr}fd$C6bi=%$ zJG!9cLCJ%X=Zls+3%rE^Q~A}4oq*<}q?j@>s*=*5HXnM%S-0ElUPZrwNO%PZE@idd z>WwLhru$KUdOkdaR8%-de5s{>ZPsPn-5OFvwJt|UfhUpS_Yrc%2C{0*ievIfU z2KMpMf_<&qYyR0YhB;*78kP*r^JYQwe6gUCH1{f0lIF*RN)npqa|O+FG|%VjqU1rz zgOaDCpn1Og<~bN!M^!oXr;mi6awD+4NTDM}8wOq5R2ynUqmq;3x(hnqKkB2rIzAK5orQ&-{9~LR5)cc0L2n-be^anyPw{ZrsEpG_%_X#8;9x z$0t9&Ayb;ia!PO5p89i9cg6DpE@CEHcO3df?aK}46~>x#uikZf^4YI7ZrJ^(M``O( z*lYlS(vLP9IMA!7ZAnL4K|c!=7yWu!s7|(=KDLT3_E37*1L0jH>y*`)TwbMFR z;06lQwU*bjK#S@>7D9BaQ2zHm-O8_51?g0)=u;W}WR`TGM0jgr4;$`?8HzN-3M%0` zL!$M`H1J?Nf6*6z>0KsSk}o*`9eTLI0UWM10Ij@e1@m~rex+l*yXOG$wJxjWWFJg` z)`QDtC;go?U9xquu$38e4`o7ks>NKLp!J&oN0J_gi69bK_VikYrgL@Wm@f1 zV#R>w0o!uZf48STydR;le~-;?+jkM!O#}BfkWi_|3{EO+W`jCP} z%GBXw_#ClmrXyi8N-J3^CHsQWc);^?L5-RT!?_>4f@a^)SCsera*{C(jSHHK{s#HPnl1I6GDT`Rj0T z+EmlzFMEMQNF?DE1l5?NpK=OWV*r=DxVG)yaNp3Fq4%x)Brm^x>0K*6xMRVn3|t@= zP_;=!xbdair7t*oGivDsb!S$7b?LddZA+N@8SvKZ>yMe*_JZkXsug?T$NgA5YCEfS zC>i#rvill#MdXk!1Y>0c-mv?QPw05N3j{f%#a!KjDxG;1CFo*yR#~%z=J$l^j5V8d zU*$0q>Am`lTD-<)N~GiVM}P8y3AGbhJR}CsG-V~L8?Q+%>S~S|?pF5226exj=Yx`o zEXhIR00l5ilj~^wX*p$yASoTQRFaIAk%9OIBae$`o7DbnqXD{~ty)tJ%(~V&;Etg+ zrogRGd3Lg#QW5LpsY}^f=(1Y%WJ=XSWQ66VxL2!1H!~z}Xf(Bd)fApxeWpZKSw_X| zdRjeg&VnVh+&(Qaz$ob$rAM{cj)9`f!s6Cb+AmL?L;@l5bcuPHAAI+ge1KlVE8Mti ztxlg>*e%goYby)hDqn3;%}-vVo#3tB82xr9Sld4FwsuX>(GJ6OZx*GjPwB0N#FPn8 zOcyNL=FIpto`r?cSxfkSG%LIFw~m$9U9bJ|KA#5V8cetn1ugUmr4dxJc6>IwkxTDq zcXACV0I0r6c^`BntalhRq^zi-PA zj1@5V$IcU=e}DUTcITd6?DhaA<{Rfzdb_*^KZE{*M&)e&r5e|Fg0^89(I(gg=-DHq z+X8|nv6mQD$Z8obKqn!Ld|L-?{!15)OXi7yX_ahO* z{|shmC8A219NyC`UH2Ua2L}h=y?Ukp|6Be42Zt~9{~tVm`Q6KdKRkc=-HY#D9UdOO zc=d;a!{-Nwhkqakn;D|NRI;Sv4+jS^%?lpgIP1xhUSE6K(Stdt|NoJcKh@qZLGOmz z16D~U3^FD-!)9w@pkbxC==|I5!SK6R!{__#6U~a0b=4U}BHqtO@KmI|M!4&Oga3b=VvDUFDgUWr&PqBI(|G$Opr>ER)VGr zW6;uS$^|fMwcamm+cL=~RZ7MD!`~3iS@3!WaIeI;O)yn$4b05lKG!WpQpykhk2Dr*ONH?3 zb80k3w~8B}O=d=v@dYETBh+e_OAQ}PbE9d@>t#B)Lt650@J9>K;`t;`eyiJ-)-#dP zlF1S@HIZAUO5kanYdNF$L@@PEQss^IH1wg{``+}o+bukEvaje^ueGKQ#ckon0d~U@ zpGR#F2+^^$g*qIt{t7lW_2SLwvc4$PUrK+JhL6nkebj6Mz`mhB6+DABSsoWW$+f%6 z1b)kEoF%0M6_iOJ%$jfZ7>+rJU)GB4e1+4T9Mg=Y$8?e4Zp+_FeHA%SA0g;#H4pFk z%8Owp<`8v!S{ylGZF=wwt4 zi%3w%9gwT)C7as(ezjPD?<>70Os!zB6YKM^`;vFzj9ew40M3$IcPM51Qqfcg*{+%`cEt`Lr&;I)C|F`OOXqse+GgvN@H|y$)O5lZUEqef| zWkAVKdlrT&@JvK=_5I_@N1voSk;n$f1n|MCH-mS49=QL-bLfnYXdybJ<-h(nIT#+k z^xrP~ZeD#M@vZ4>#jvYE!a{SsI7{*~mQR$?{Cw3R=|_t#`%#ucDR<7SgwW4_ zt{c8E?T$>dHJU2;#>V%p*daCZcS)K$8yz!}Ombjo8s9g+omG-N`9|Hc#m3#0(2YUD z&A)8SU6R{S^{n8zlB4_T!zgKDrfy)Tu79KdHF@@wj=3m*zoR05_Scm+C>%yXU&w;2 z)cZ}%@k|wUo~)!lKPNmQ*XY5j`k1x1KNoYojJs2CW^cMa9NX#Gwrv|7+qUhzu{ySG z+nuCiTOB*;*l+Yz&+OlPdrtnfPp+f8YE`ZE)N|=xx9n^tmndDF8xunI+5Lj8JI_~F z4l_ysN1nIEz7?Kl-4g7#w!(O3zPDL5sxR$lm&@aeH2Jr{sxRgsA^!VWF)ahsSmR{u z<>_KrTq4ejLN`)WJro;^R;Ck(!Eatj3;lM&ZODox`VJHE7XDB$H`oAJ?6|UH_)z(#Pj(~2rz+KZvU3GX^Ze)^L6H`PuOGfl zT(xl%u%^F=YPeU=*IusK@I)PN!fJjT=SQcR(v44oVu61EM;^UBJFnodoS$yTsHMP6 zy3EknC{F6oncuyT!U?*ePo!rfh}cRSnwroiLRSY|qR6csS=TJpWZhxkySQR8h#sf> z^c<1`O=@013EzWbfHPv1uQnbw^M{&ZZpxBYNt#muq(1shoekpX0X8hFG7IT}9HmL( zG{pu%9%yB6XLnkHl|S}3Qk!4pM^iFY)8UO;L2(=7acKCCCVwt{Ub{qbePDRozmwlb&3KW!+Q2cEZ!4i^PJE4uB7Nl(p4 z*C4LOt;sv=fOJi?49SBi`?zUWl^Q}P2Bj>1kxH28-H-vk0{&1Ps`Y+Wdr_fVh2bu40UP!RaoE+Xr*NKQ zmtR2DI)gXm5u)PE4Eyx$5?qchWf}cHCUS?6@{feyv=I*#iX&wha39h&7DW{YSjvy| ziB1vh1-H6h+S6k0MVpC60UAdsF^BCp{71#d=!j_}vaExygG>9Z7en|Sib7656(q^E zY6MQVFMjZ#u*JEFG8kdEMhcHa=-VO(m=bg}WOHA~&78WHAVQs5Gkoji1nKs$9c!iA zTY=hP2{T#=CtG2R(d@?6xg0WH7Q|M$tm1Cn6si31!*1n^OKbbp1}9Ozg=C?_Y@@@? zXb(ecPef_ItNwuN)lyhgabjiD6gHC_^c%|#=EQf}giv`LN`?`~2<3Ov9@mbS?$^Py z8s$6@11OWdI2g|N*00xOAopjU@<8ad5|UO!uc3KOSM@^-vBaUTuCKw-MiQlp63w-b z9@eDa8{Ot^Du;zu`UETVunmz4Fk;6zvyL|b%BWtwH9`%Ym@>ery#aR-!`t=f@{!LJ0pt_TCmK)<)H#ZN1ZTpm7`10l6%~8zW!1>Q6 zS$(+8Z-UI)uDglPPM$8@{5Rer+2+UjX*J}yx@pp-*uqN`sfSfZid`xZdl-7>eriH` zwMWokT)?wq2SdA%fa$at2iD($ce)wBhj6k!&&DZRHia2?E*oBJ9sDehwkVFsTcQo^DmZV)*Lglg-gPtZi3HANElM5SXv(A7us@FTTPa(aMFOMhNr7^HkRKGMOxBnW>3=HvLzmXYPt=rz zEMM-hL4))CX<*l`uGRg!?FR7&pj9e%w8le6aH!$ZMmaNU54oo5fB1E9-`AcL)Ps&E z?$JPguX_O3tThvIovaRAXfW_6L4Jg}NccsS2kd3P9msFUr_DPJ@JHG172u)+CdM>} z&A#vL%lpsSTAT*Hn+CyxZVhnZ1^*-hWm{-JgKq6FEC)o4W1#p>#ro(A=B2_pRGa3V z(9;Hns^MwIr*FRXUQ`_*olto=qTbjt?m^aKx~mvUEXnvox~nY7WJ9_yf>a0Cey1*%Cd$rSVRJAyvU5bhRyEH`K^*T=R85~UG?x1e}Z-W^%*ZP zcP8~DWJQ?ldHopZKBu&z>aS3x*W^;qGoNaP_LRp10N$spyqom(!rbpGM}&YEx{1lk zIN&00pa43d`Dxx1+E_PmzUA@O=TBG$`uFw=y~i&rd#$$`x%hM;jCEK>EB@&UQDUhL z6j4&v;~1|Y%L31&!*BtG;yGZEi#R!_+T2*QJnWhv3shUIn2D;sD@myz{~+w#B)cWd zyh^l-9}`IGlsLmf>?Su%uEv7Cvx(OenB5I5kAPmk*90=v?^kcnHu*}Q0u=Z&KtA*q z8p`zJ9l`p$OOg|jsN9B(p16`^2?$VHLYZbIU7;x$;1qQ19GQM5*K)P@!fMl9#X)6T zk5(}Ri5p|C7P1>16pU@LChCb9KDOV!;abFbZ{1;ct*A1)A%=ivF%207T{&OY;K(w& z**Eexd4VA456y7|idZkdvZiQk9f(?o721AQZ6fMl7mM#GYFU6R#gMJ;1B@6p^Rp^rJVcYH z(^o;@ZIKVp?k>V}LVibsMd2BdTW5#e>$q-x89e?B-(t$tSk6T^?_f)@>Q-D}4q*}| zHzlTpDgwem6`2s*G#e40&p;KOOQ(5G3g-nR_}E@JVP{;?q?ba{Beq_wK=VI0+wq#Z zX&kQqG4@&KAB{1bc)IJV89QCDuy^*)!06NiVnEkVG*mSSj|eI^#b%*2SbbE9M5>_F zA<OA4C zCb^F=-N@Fu&-^HbFZD!;=aoup(=|pgr0f|+Za1_Opyn3W0Zcn~8{Ah_a-8t}7@Z&g zuK2$E#9kGJXCXu0ak!cln{^Z-do}duk19$hc%16g@JB$IA(!;S8yY3FlB?2v_sZSS z$7gbW7jy+1pi4Djpz{i86|O2*mlNV zA|KVtDW2mVmy4q~1T>zwIMZRp$QjZ?ZTUT*{|MD1XLOC~c!%5{(jycc8{VT$56|vr ze&e=Cy3@;cg7O+2K?=4OS~l!5KD5@JIpHOmTJt=iejl9ahBlVP`~gxI05mK8=U zWob#v2*Iu?`V>FYpBh_+d)$Gj&nax#qD}BmkzZ;iZi+ahA17YjHbu(HbM776sIBR- zHRGsz!>&lKQdsaqNLeU;_^~|+ly`ZwBW8JcN&nvG>HorV6ybHIZ)TE!t3b*U@4UM;q^3& z!T)wL>0;Sm-Whq)BrBF%GkUo}V4$IYaNn)HDPZrmv^?P9$-DWxrdWOU-6v1OUF(Nz z7O#Y41v598P>Y(vIu4q=hBeK&>N@t**4CDirygQ9=J+Q^u{i}cz9wrT#boi@Udz;i zrPy6eUqEzqWMayqmUd&;Bcd^>2iJT*lP?%3-*Fuqegk5N#$6fafuPoDC}k(Ty+?7@ z1u6t@U0C7@mfRXk*-fPA^{@}!hSO2U;7uW#yxh+gi2s4LqZSCymSV<9e1*+uza{g> zm(eayQ00U;ZEZU9!3(813OAcv!iX}$=AOOZi_i?+zDecAK&m(e91eZ7Q0d2IintJy5&?n8>)}&Wxd5!H_zuokIaiNQ$yJP zBchx_^i;dl3nX~nr#|Ct>yUQg%h=X(M0wp3Mmfzow@rS2n5>UXT|5vO`T@m5{ckPv z!Z-R+*+H-E>?_5Q_hfUX9HP-KHb@_6EiH&srp-j(+*QKX@0FYe^=J2+(OW<5x!47 zOeTT2I&rR+c?@WGoP2<1g}9@|V{24^;=A^YTG!R!eT_^rEkI20xr7`u9g8Ud8S);w zdT!-Zn-gyaGoYfuujIE_sGe43m2uE^=-ox0I2}#JLG`Fx5jo`jx^eQF^XRNQV&*s; z!E5fd3kleaiE)s>7L?`^YpylC6Hi)1hoHEog^|5tg1O&#WexSdBwU@*iG>gu$C@M) z^N~<;fuMNCO3}z=60=?wU`)|~k-e(#(wD3PaU*QX(yqkQpQtToknDf8td#+W4?L?SP0ULA%7NR+H8P)cSzRZ8Z)TzZ-Tdgyf*u2{S1AlIrC5C4 zZK2j8v>n5AZ`fk)=e^t8uOV8gIXEZ0}&Z`q}LSmSsM7x3wzmZaczr6 zR~+@puavqtmk5JuxvSrgn)~~5prbum`CB@~k z?xLW6#Feq>oAy>cW6lC3h za_e}@Hy{qti`bH%CMbtI7Jb5wC@Bz*Lh=Z3-S!ba8u`_yXw$u91?O%;G;qxSFy5Py z;wVYdHy`vih{$2bP&8KCD;m{;0Cw`QMXF5lzMEx)?OpW)_wzfv{ODbhjg zHq=OWV_gXT#cv1sm=kVc9Ctm9kj)pCr1(Jp5fkF4=;B)`MFk<;O+*~}J0eQHo4J`c zJa9?t%ENeImsV`J!{rYn`N5D4SbQyFTDYoi?@y!G7m$8W_qb9OXq~VD)BHGjy)?O= zP;p2;v2Ltw&)>v9IK5PkdOvfFq--1%cHLa&wK%u=Z0*Qerm@#zf_t%P!wOYqoGgos zfNb=u8ABAfYWQ}9>d2&1FhagL=GICE2wMTM6AOWs_F}FSLx&B z)gXIG$0=fH2PC;=hQ zl9zEQMJ=gMV8p7|xUm`5y|IC;*T{mb*YJ3gC7R! zi7`zIlOZg)7kll3iMp28FnpoPSC6`uxnb21<4gyWFV>CW6kBu@g*SYTx2GaXFH7&nG{u@BZ$oen%NGZ7~6_!O04H*WnA*+g;PoSAD=*}EhKe2~ zrD$b3$y)4Ilci{^ov>K?v9Sf(u`v@k)+T;-vdORjsBZ?33Apt}-~(0&=tvt->&}Qi&p(sPOaFl=o~#_*j#XXa!!vM_|K&1K|H^p za@Y+JroJ8?iem-~!6rum5{kn_$yDkU2*4#`>-ntG8tX&|7Y^XMBPHs`W7z-laC7Sp z8f_Ll57EqmQHfM53Q92tZiwP{XH(9r zWHc! zN@G^_9P|E_qz7-wQixpFR98r-}5hleog+y{2{uU&9xoXkR@CFcdh3ZPu_`R}H5n}yPsW8Hw`ujPsB3+j|T^sXfuiD5{fu2V@YyJfOhb@V!j6+}= zKM?S`*MDpbNSREqUhxKiCj|Mf#+>i)U4F>ljmH=E_OhdTJbg}v&bIg!%hXjLy5G-4 z&3OGmfIlMuV|C?(XLa2GOyyhwrg9=-cL^XYv`x||EVR4eO~9rmevs>sQi7O(y}-*> z&!iW&oD*Wn(*1(C(Uh>58Ac*wqXKHq_eX!KKh#6Tp3c8%CT6-e60@BQS|w(>22_kq zMNQ=pgoymTAuRvmzJ*Awmu0Jo+3}wo6tiP-iUub{RJ)OemJt#SI&hkgUs}S z>W4>mZdm`Jytw}-<&iK^-MO3tNaF4x#c(ZEgGu7B7uJ;f`VvM)rzX!V?^XQIUyKD< z>-lJ#j<9>pE>NjzvDBJ>8xU;ekx&SNv}@XCTr~KJh=qc#t`4zS4d@9n$F1DyY0uQo zAI@!fvE#EEzlKM{QPUPw1Qmfr!=3JTC%|6BSPWK?Xp5`{S)QCUh$M#LYtbR)6=(o- zU~MQWQVTSiCUsz4j0$5#al-!PWGc2&lnpVVHDENQu?5nSMJ8SWAQqLAnCWSkFc=6u z+&0MP**8@QrAF*zEb&#bN2RO5!ei2JzK4I7luxFVx>=0yY58InFWPlnQ=-vU+c zbi4Vc7A>PJef3nln@#S|l^r*)y?Sg`my7e-(n-kROG&aBF#3I3*x1c#l>ysEBWrWb zEs9DmXbReHrutH!V*Viy7PX9@2*_)SHtrz8i$-wnuI`$?NY+l!iLq((OF|jbXf@_C z0dR;ujh_jTmyPI+xezFg)54mcGSK}5h@Z3d$6v@$fv@pJGOsr0|TznNU1 zIyt}YA7-??LNZaAhP0f0P9v5|47yYxJe2>zPoSJil!oDDvpe@jIglpQqAeGy^R1lTh z3Fr2W#LJuV-|4zfw_``PBQ@f!Wws_R$WB64H5BJU^>11<*60J2#u++L@Bp&aC9|n< zDg7I+^`epPT(?dr*;lb)rrxQYwJL9nmXKbF_nKahRjTcbxp_Q~UW(W?Y^E`|^Se57 zW5ugnvrA#j=J$M-f|P%iypPu=wmMVU+r!6H3Et1YR(_9yn}_pdAFQ{sQR zlZe0RO5W=lT=-(gU5m8zs?yv? z#84iEu!UbDk8PTSDLGyIWE~73h>7|TGMX(q5-{P;_&NCHrjYeoh6Xxa{oG!AMoC^sujy>$POzFw2dZ-Amwa(wH zo`pCpJ@1+<3@b-En z`^v+5pY1eke<=UlYqJjJKp&P*rz(;qINT(G1y&!5=(pr&O4Twe4B9SV&wws094c|J z4qPR%QveV{abE|$fz-{VYvd{B?+aQez{FI;F;x$@TgIuNSM!qQmLH7Umo#z7Rrk#0 zvR})JR;|BqH58Du`pb0}0L&2XHQ{y{?KSTW_EvhW5r7d4Tum#zLg!ZJ1Dj=AqXI&{ zW9`D00>#d?K>UxQv<(ggip>ip0%G(uHE%P6K@-vpdN0PzzUZ(Je6_YUJ$Vnynm5wSe){JPOWKpcxS0)Ly9N!c!+5x(X(Sdj5hQdlpc%;qXckGix6a^JLq7TFGv6hI~TSyVzMHsvoegbtQs zJ~k9nNQ4ax{tLH{p#j3Bh?%ogh%Y5=%)zS&dw#2Y{E6g6Nr*{$O?s5)_;;Xmm3O1p zmGVVGHt+Lj*orZLn55wkX0l6Cxu;@gFhj;=clgRW+Cu&Rk9E zb;C@}adCs};o#`V8=KA-{7YL1j{8?cbuqc2t9zPW12?LTE`yYWk4#74ML@0=$OdFN9kM z8`}0+#JI~Q-exF7k&!IBHpXLq>y!Y5kt-o=hA$_4}?!A-^ zH{hC1P%r1v((%y`5$ASLh|e_z2n^;lEbTm67P9^p(H9iv6Y4Hp7_TElyd`*p9$vDc ztMg7_AtB`A+;)>Gq?ZF43rWXOTpEbLOj5x2VnOz=C6;@BYnK!Fh1N zZ>}0mY+Y4!=;K%5T@k!A=Oq=%MT*h9XZzg=e+b8~bRoyD6hxZdA}G82GsK)1eq9=( zFeZP-4s^(mh(|x{Sx)ajMT4Q&X~Q&xwkhI&k0FIr^ssHWWd_$;c|tO(jbu8EA1z>T z3n|9K-`X`S#lsJ2%YqDP)5{$$sI{<9(fG?5#&ek7enoJv2lFqsPXxm>f7!*boLl0c zUVHAv)g3dg|DVaiTPec)3eium!}wK5G3FEaaHE+rFfnEph~{EHf07JC#=ssnNlS@v zVhUj~ZHEA>mCM}@P~-P?9EYr4JGUlO)U~Zyk9ywhZTkM$^AVjttX09S>mIYXMBj1O zEs|Rw2aRT{OQtCu z3nUff7?*aITqtpRjgt;@=pOv+m+#q6c=oh&Y{n<{sc?n4LJ= zqx;&a(AUXq|L5!3*T=okm*v;0kkH%37lGhM`r+&Kvo`QeV0vNa>#O+rI^b(`vw7#s z`Qd2_2OifA&O#;t?%)!3!EJQ@nmXNPLSd{mFb6(Qg*UrGPM@= z$K^}hcg&B=m+2EMi8w({%?KOQvM=3bp2KdU`=cDVCPV)4s+qO+$= zwvv|6Fl?3!$^r>(C2kzAcfKzFOpg}d-#0HF6i+TbUkef%_V9S$U)?>N+AU>5EC}f* zp07KBZRK#uX$i`>l$|ux$TF(UK4%lY__0#If^1-aqhYA;9K&bx5QiHhfALHmD_Pp8 zNz zI81GPwI;j>cn&O|;d1lmTlTK3w|dgX;m~aKe*64d_`;^X-|c_;(lk8;Zw+0_kC1iy z)Li9-nY)BTpZ=ig|LCm0 z_;sQ9!xG{AxI9qPrhEkkTD)mYDo*-$H^siTa;imq#zj%&x@no9$6>hZTzMOW~ z)yvM=f9}BPcH&qynZ0;kUh~diNvhWT;3EJ1;occ;jB2Ib6-BGcYknkcuSjFMSaPRxpU z4CFEl@BI8r0q$AEtkWjmW@n#~@*!jF|3v|b*6-?xaz2eiV-#^$NN%15-QZgUpkzJ| zS4f9R2J~YTz05j?NqI!)=>rr5S!bK+GZ&Y_-vbz$w4dFPTTTAS#P)|hNwPVbwAI)o%K%#s$hPBW2-CsejrKyFsx$ao{9w*zsk5fe8vnKLaE-e^-`PULVp=X`I|iEdF*+XK020_PNV0cj00(xNe85s3$eL2gsPmrL7uO7 zDI?RYXOLRe{G=q6CA-{@0734MPMfn}>l|Qb7V*6Ev5B)Stea@EURoTu%xv{^wKyi_ z(xGzlGv;4(#9TV`)JT@Xq&%tl!w;{jnyamCf5zX&>n2Qb8*CI{u-VMyl&c~=9h!w^ zBl`pJ7pDPP3o|yd($k^;f2ru9D*4D6w(2PI^ESCk7uUCjvXCRd0u3JKJd|Lr@pP-_ZD z*%L_u>U)eYhmI_|hD0`18AzA8J;xoKor&8U`xYdX@wkC^ywp@4q*tU^m{Ww%Er!M+7* z$57c!d6!-#;WWHm_BbM{@t5t1(W{?Ba*NwixrsKvG#tbY2-yZqChOOcsO88D&j@6rID>b~RFag#3o_Nr~*>PhTTTRMamiCfOOP?@6h|?NlW8O%3m|c@yKf12e_mSm4>gbr&$Hw>vG;2;#H6mWs6hK{m$7_c`qreklZ)ExdBD%d z`{1lyBSK&N8K64cpFT~(&&~J9FV7x+%JxrrMEQ(1gGh4PN-%MEAjr+%ShK+$1;}tm zn~}}pa^&Z-1#Cp@RRxj~`=@9$nVD2P5!o4a^Wur6hiXvFT4jnSsEXWsVFG6ls2ob4UMkErtD#4O z62xT#hFt8&7n1($s5CNF`5A6{x+t!5`bJ5j*5A1V9+^q&q_Jo9nN!4BpCM2p=@VUF zmQTuufaE~bD}7}t+dfw-Z4X@!SMPyZnQB7D*Rz*$O>e>5HU?kYfpZirVEDzX2bRnH za$B4+2><=L|D&%dcK1Ts7EBdcw zu2-Qs=ZEUk!O_vHal+Ah(AGjcSl!It`L5cAi1)z0c!fRcQj}>-l^PQ)$aCXQt1^X; zQLKDHo)fjT_PXs4Ag@5pNo#XmfWvVFQPQIAU~>umvWUPvB2TwteoFIPv9uG-^6l@_ zHJv>6<%zPTY&P^f=8G&~09fce0YfnjHzq`K4_aQpg^Q0pIh+^DtN!Oqn~ww<_Y zCg-5|ck`Ofa*5d*`(J#Kb2HOTd~-9vI!9AwgzLSlJ14I>B)N?jjH4;T?z9-p1rkLr zPdRJ62i+h&xAOhxVVIwXgjohciH8pmjSyXMCa^E?7M31;2+eHWqpi&zbRzV@!*^q7 z?5J`ejS%nuyUsL$eW`Z9l17ScrbvDqnv&f*Rhf~nX8}$N*(%7IDT?-!0x{<6BLjx> zPz8MY_hWqzbKuTBqv%IZr3QuGRL~l>cB;KCFcnOS158T`EsG=yVY6Bo#vC&Y30|F? zSFRz>UU6CXhtZJQsWgGXoFwu8GG1+oeRBmqq3+T*$+@=xk$(OM2{UJn2Qn;6`25$Q z;Y?rw$n0FaZb`CO>eX&`90)Ts8Yh{5c7W0wo)7J*>Xi2Q%xP5}x7j_yDd9g(i{`u} z=312VyV<=RhS@zkUw5p^S-W~mX|B6$Fes}q+6Lwh z?wt1+S+(lCQu69mw3!MODT)Ft+#gR|oY@&ZK=z^@7>+Oo8kcY9x8JfjS5U>Lk zj|gOwb0Nvm_oAs|9KEtI+J+6`2~y^6Wa(91a0hc`xI57vW7eZh{XKhQ<<8Mwcd~cD z9`Tv8odd|(UAx+Tbh3GG{~pE(ehx$i)twCdc8b=5^7gn3TGL4=uw0$?Slz@h%%ETf zyRnJb?df3aSdW>#s6S>k|N5(oe!7|jjabO_I)S+c6Fik!X^UsEgoCAAOUVH(`PX~0N+b+gqw$3K&CLjQ^~ z*~o;N8L`QQ44I_^I5Duv<3;WWjxOMn0;k@m0&q>G+!mwxN4V4_hpe^n-1%U&*>oZZ zSAyyZ{I^&!E^5+&zphS&k| z??rhW%w2|0+??g1L!P*7mjM$Z-w@k^a7ULC*@AFSwh-Aevvfx%-w^vRh3b>dIp$Ag z4B@#ElmGJ->}>47_fnjjQ}gzOW_r&_SxQ#Di|a)^LS9UuL5?x z=m%{Y-@Lp}`p!u?x|un9uRQBWHM0^a1r3MtlKZ%L2pL1R0)8#kjDL_4D4IAkk9+aJ z)Qv+P;rr*qnGiBm;OnDM-j>jt?ZWnAFo@y=0mzo0G--$^4n3#ou! zF|j}ngEVr)gH!i6ryrH_eSm?ztwq4Vo}r-=cNr@Od>LyeZ%n29e==k>%D*puVW^M` zq1QMMu+97s&H?W;bc8)GkFuc6VlGw&6Z49b~+;d4Ji2(kV!hgz|_88`3^U z@Z-vfjo}pO3;25vyJwjmyXWYf9?J1Q2PmX{>;PR|S~T?kc+`?mNMI8gv7>nmnZNxN z#?euQ8Gn!3@VkplWDW~2-2zIf+tflcxg2k(%NJDnMP zb(1TI_(+^*%<+hXqRBLqULL-C*CPosoF(<(ta?_@GRq)EMC>3U6SuHhdg=!o;qH8L z$lTp+rvH$Uxl2;OwD{Ilw-S#;vUv(oI%j4IFp3k{&<1M5wnt=8GDr{|@lzyQ^AyS8 z%D%Xl5hhGS^J?{<%ccRVO@DLXta()S&m2Q@KX)cm-u~{K93>BucGyF>h`BvEnN;S`;bsn!E|Ll3LiQ|3YYTMsIpV@3_Yw>wZt1HchKBeaw(GYdZa> zfdainU6ekJ@Tv{7ZrngO zVG_fgIxsw1Gtw(8@_!)aTC{J1DXX4U{1TJ{Ix9DS8*21ugyo&9;E9-rl2)_=rjRr= z8iiK0y9$Pchw|S_UP-brc0&PYdGn#EB{8Na1dlACMfj)jM}^sVGZ4oWO?_b+q!V_o zc$%4G#may~x44jGX__xRh{9ETL6Y2FFTBOMIVbL(vF_;u*@crA&LZIX+tmArUf1y? zb9$Yx9-l1)%GA37eLQ_G5?epZ{idOkj(Zq&zUM6P-MhtvyVdl~v4H*ud1lrkyu~2nV zO^ANAbC&on|Dqn7uZ|c|jZ6rL;D8=*didiq#P#Ob3hHEzIVj*8vA5u>103VaI{7nLUe3ndJv z)p*3ZKUye^9Cv4{i;O6EUIS`_>`Qb;GHhc_hNwAXP2OmW*O-sHw3&}LmbGI$9sY4Q zBYJ;*P=XE4|Ir7P=tx$X>ocbX_8HNC+0inm5t+XJ^+A(g8I@_8NAij;gw0Njs~VSz zq?+nOO&v4bx_mx8=9sG%-!+SzmKFFDJ>;cM%G;TU!hfpw&Q5M`OxAa?Q|u_;N0oX# zC7?SN7Su&_5uW>0Tol%|H+K=b?DJTfD=7Au`M`za7T&kLvbt@1#M@djAOi{|V~!GE9a)xO#Nz%- zNQL9RR+F}UnM9y&5($W%;)u?o_=JN=K0tbfqihED5x0E}TA^+R22>bgwe>q!*|aYq zj0r*fB2s=tRVQ-4=_)`H%!XSUP=>yyHq1x*Q31?F2UGwftNOK2)OSv;Q6Sy*Sw*hO zqBuk30h0(ahOkbXyfOVwdLO)N@L)!+>aeQnbb>D)9%Zhx$4E-+TWU4>X0~WCv(8CmQdANPRjM4{9IUVkB`Mu7Z?0C#7t?j{r|u*Eu$pQFS@!N>)Sl z2YuWMM**Fp{C9ALCQ&pXDpJ-1a%b;6c4zsE!1|k`YX?14Ro}}BL51vCY7)BRr~dpB z`vwD7`z_T@eBzCzd4f=a8EMjm&k8i+rELp_u} zXVICq_h`!{`h3z+&<7vD9^^y|S+TJ`Fivviqbc3hS9JpqxmSuKocS$9dhW&zLwH#+ zsJKXtZqPKn9m;SV=rLw7HyU@QiW{tb|9E#-n=MHeBR9s;DR4Hw_e{e4;Y0vNJPYhA zFKjtYMv_PGm16bz-od*LZeLLQ&yt;WLCIfRIw75Nnhuo&8Sh)O19{x zK$8zkIwBv3wa*I>u-0^1I&Rbnc{Xa&%N+V+<49@tl;v|{pel3KW~F9b*)F?fU)7(u zxBZGh^;>79n!o&reas;&3lpL=hqD29i9CBH{P00%=coJhf!}8V@UHe<A0FsyixIOxi&2g({ zM?}$sN|+*c!J6(=X!F!XHU94SvR`PJvHY%~PQ%x|-<_>5+3)s{>NNj>Zl%6Kw*NT1 zM(1E|WVMxaJe|mXIkLQ_rePSJOynOM7?ZC19k%&JV${p;B|5MlblkxHe@lNrP$8RX7 zcaqyD-SbW`Hso0(r@NZlgX?WM&g()Luv{f77@dBL~;i zX7CANQ4SY*jqMkJCJ1KPOSt5)uKv< zHSCeV>+5hrk7oBHW1?Mr1B3@7M$Pso-Gy?L_b6Vo(Q0niV&@ZXfhLFioSiGRSbwE- zYEH!ge)x4?MRwEr5`J#}hJX}&DMb{-K6vYmwzp1AK-ZO9PPT_li0zXCvT^78yC|Xv z(m2bkd1@MPa_3%lDGP{xt>2s^S5nZ~$Iy~i><}!8S`VVy2U04j(AbpfjX<3)lIdFE z-4+4JqO!@W@Gd&;L=63<#DAJM2~Q-?PK>dlC{xQfsNnOjUxK!mx8oBIo6Nq|9oIU= zGsyK-fPX|*0DCwDwB~Ibz}1JS4u7wBio`z0_-zfAIe^$lv}kw<+V(h4YhD*BKJK2r zFZIf7dU{{-@mX`?+V`#maFW@ce?ImML6ygO^AIEjwqPWCgPjA)WlG$7yU7(~rerCb z$UP^6X!=!i6CX>+U)^;Y`LoK~49G~$ysbvzKVsV)93Clhjlc^nzANPRvPIawGT|>N zGaO}hAA=G8rW`EXwO)I&^+d?Qde8cKH9sOb8DaHk;%S$-1JxcKKZa>8##4Cdts%Bax$C61s>55B)^^ z#@5w{1Ok0zVsVP5Gc(xetPo-QxhJU8`#V`nC-@7VYj3WvE-`Kd&I~%o3iiXv8omT> z+QZ5HJiP@!>ebIfL^Ah`@9l(2qx1w{RPA8DXHH^CDNEfQ)RGV{MCM6x?lvpYMmG@v z_HI6~(%|}!ROYqRkh$>1mDQ!T-@I*TjhnCr%<3fv%$Ve#2F>cB6`rC~I5SyLX;W|9 z@-g_gZr!dKy}D*G__xMSxVP3>op47}>AhD5V{zD|-8XE?wlG=AAQyj5+|8Fs;wEo% z;IS+07k zo=*5Lz_@U%f>LFC=wQj)clXyv3wHC`ay+!m>}+`o(`{>EsLj>kk>HL-36{t+r8NOdCAmQ2;)zb%8 zpmx#;lK6S_dbiid-5*wE(!9f#{#9;vf*4}y__!^0-y=7LWRmnmF*!My|3 za{2cyzi|_Gk{%rzzd|m#LPqt?E8yYh?so&U&e-fX^px5A)!!05yD{^my_F@_sjeI}COKJXshW$h7Hg|ymO85Q_T5pxB!xfKv(KqSmKT#7`@6zm} zQ;rL#-NEQ>{!XhlFEWg7{{Px3H|T8=tn*zP>BwoJ>Dw-AS_fGsv5J!Q;;Q6|OR94X zZi0g+S9KBdJj{`ltuLoc9gx^X6 z_2e8BZFF!<@)&;|pPK()J<*=nj!m6_o!tMhv4Q(}q@90SMV+R-+|5NOzo7*O)CO<) zP))>GUnSuz`4k%bf}7*h|CgC7q5Q?8l?-1XwKXI*Au^JcUx@z~K4(=4VTZF+m-yMU zRCidXFFQ}Arg>pu5LsFOS{jDtNDQv=>E$K9X4j~mgp5Tg;hL@9YEh;?N9vsW@AkR| z9A;q_2p}8tHhxGY<5>;QUrCj^&%Q_8Ew|d&s(yl)M_JaEf^jMGy|>+YO&X+dUWQtK z3rj~Uf@jEaU+FXFo<3XE-VBQ_P;a!&K+4nnP6A0RxIK=e9+0yY_v8Dzos2?dJow zh7NTn=ztqofhF=H~+j8pREBtqR1*JBQ>|wks4MEk$~%;%p-x; z`RnuAoS^=tF{DdEfc)_)i`2aQKMvrA6+`zPGjyX|dT1yd3yn*O3>Q)Bm_^4QZCe$e zAz~<>!UfP()dkRSf=UZv#FcoTLJJ_Yvv|8lZD{$d{5!8ZOG6ou_D^5+eTEK}Qj7SQ zWKt32Ua>G@T_S|hT?=MeG@b2CgB!648``yC9^Tk65AJPT?xTMIyFBA#IMac-tAsTz z|1M#~yoP7bBqrhN!Y6q*KufWP5V0JDay+1}dmMY360Lyc?env5IS+u1IQB<~ju_J~ z4VitU-?gogC#Pt|3`_b@xOyiIeZq=u>&K$xdN@j*ZD-rSsFA-vf zY1L+eakDHDawnVaF<-C|lfKV^p`)nTu9y2my96N`6hYc8{)u{eI)?TT9))KrZjK!uJ5Isn>RA_HH&=O|(-u1O%Au6^SwnfiER6!@t$ z&MlJEXS_rrzFKz7!gf-2tQ(VA`xw@_mqpn>W0NI$3Ql9O;UHK6&*bN#0uVPzT1|;j zzw#IsTJlo-_d+0y0yh028M)7A2C+VId!D%j*vulSU~I-JJ_NlgCyrLlo-f)i$b4v& z9TOW{)ji^A$0`x^CCaQdw!Cq9r)c{B$@4fou6Fbh90PA__*-&+(^L%hzIuwQtUoBn zFMn@sKTcd%TYL6d`_p0hix|RpCUgbP-*G@)7N0)hO~aoi3j74eLwW-F9=^HCT(X@# zyU!BT6!v`r{n6{Pj~-9Fah_!f-ggO6$6+vq+HB3+EBCFk_f+UWbfeKS$q!<;N*ZZ)u_?K{As{arW(oq zTc>gCkbN%aGAxi5B2hN+QVHE~(CA^Q2u8*{DBTuiRvM$NS{`_lq^w)=;2(>nVsX~? zDj@TJWBFwT8*=-TC|0KVovaalPijk>8sQO@^^FBnjXh@jM9Ee&?gNPgBivl?;O@`Q zbt|)7Mqlm@&a%FUWnZ-;#TLaVF!*_**qREILIwKO=zBekw*S!l)Y3ot(f=FGLq8v8 zxG{a-)fyWy5aY_k2wV)hEclX@yoS}7&YHLo#>Fct@;gdpHM5=MkKOE%swMNLmtX$Y z@L7mR=)IvRTC`sZUL16k4E?^p)x1v(9WW9Gez#0p=?^=2R;1gE5tdu;io?WFU9j*< zf~Tm%*e=<1h=o)sHRqCX7tJm=P%0zFZ}Dl^-V?~B)4FEr-^_;17^oud+0snzV#LwV zKWkL=$Uke=^c0+g={{r&Z5vrirvscj6D-)Tu7X0Mf|;vT--6F9jA_w_u!8XO+9-&D zj`zu}VKRP{N2BLvLhZGf7Ll$xVpON#Hiv0ts}z|}3kx$D08Y_ zdn->!mEgqfv%@1OF9GJ`CLQPS@~_XByzDruTJP6Kkf0yTwU%#huY*ow$vWObmRW9< zC~LWeNz@iVZJQA7MrXo#nj^}&nxV7kiEgvx)V5fpvs5v>Osq~nW;NEh+>JB88iH>= z+k2^9N2ez~Ruk558YaxLG75#!OslQ(@5zJQvpj7&u<<@l2~Ot?Q;CP6wp-LRhU=-cPMQ1>MmQZ1TpE$h>}5sEx$r6dxBg74_~Z95nIo|Yl~~G zw9h}hk~C~U@{L0QCI^uTlO#pfh;I1785LG+aoCBfBULDr4grAzM}jG26a^}h4xxa~ zCZj@DA}JITN-@JvAwz>VR^@{7cijy2Z{)cQHVk`?VQe_pQMXHKx=YlRFay8l7^s9n zRy`!K(S86R6<+|4En zj38~$`3MPJ>z!@L2$$qclbJS`Y77))Zsc6<>;M#8!=PC$U#TcL{s|5TclViFK&lvs$k!PzYPkl>` z-V1`Vn@W$e_ZE{9lHTLNfjc zA#kvY1y-3!j}amum}Fz}bvv)6koWb!w7M3`e*Y$vepO+Vh3vt%an+HbQcxq3UD27i z4#Im$ob5XxM(p?D2<%%H5N(#%%n_kH^lR%$3t&s=F)^&_3V77@IzogFakAE(wk

Nhbmr{s%2Bb74SDPy4;jztoLv#G$p4+Y- zZ!F>zI(k{{fE+n%kR$8BpKHY5*I8dheTZ8up8Q(ip*+BKPpu*lj$X4}(wFA6_EnJO zI4o`#g(x+hI>h7`B;8Hu=r(^NmN$D>ok*~ZYWfq?io&Lk6)GTKp_h7R@r{j?K$@`x%H?)_#y*5PN;#I74imdrgr!8-bX618uZ0$=&|H!xxg@1C(xkrIoq^K zl+;qyWOiZ{T_$5iM;MhCg`$0pn43_Dq_Zjb$F|^vP8%Im~=p+SRb~{ z8<_{hUZxsRxvWWNvO|uF8}C6n|4XBTo?bpXzH_ZhmYba#FnGwd?jn)KT(&1&-Mg4S z+vT`^+ek_wZT)IF+EP1}@)$tauESEl0<)E{NiX|q&VsFkET57rZce@-K%Wc@N48!S zr~OrteN`_isikylRK@thue*+BuWw(a<=*~kfw=_rxGewF{D@=?MzoT$GqIct$@ z$cl<#587`_RCTHzRy&@>0&ka1Nd4~DGT{_K-P1~AvV~mYo1J2s-2}*FxU|vz(YUPn z8~gz!Y*^#XGOuZ%GrvmJBS zo$B*@qBVDfG(GRF5139pak8r^f_C>{?*>n2!x9CmW;Z@Tr_TI~0uNb4`VSx8ui zG4OZ$E@NQe{rgM%aosZTJD@9<-goIG>z*U6tX>g@W=e;n13dqh{NB@gHU_gORv9EGeyrze}($!hGp^1X%_gfhvf zDVasB0!vdR^qp&8`K)X1Hqws>nl-9=4x<{pxfDCJ22FrF?+x9Rn4Hg5ZTn9Jv9 zu0n{r%JwssyeDOn&i~Bc)qm%9G|izSusx+qH7l1xP6TBfZ4flb#&)lF|69|mF)_6> zsJWL6SS78U3}qVhU=uU@ylWSah+zSKz2>c%-CjwlZ_`B?G?~*jxvZhP4acZLrIh1f z;$d}HgT}w3j(#+#59TmRI-&DqN{o5TRqg>#m2U!Vo_XvkM*aZ~F+91LU8+AhtFc>)295HS zev-@El?dc|b72(6LAO>bRmMO|k`TvPT?lYw zt2T53e|XE@M}A4SsH0*;3%(GL6DG{v>Q7qATVGaao!-l%omZ zocA}>_VtV#b2{{In-C7$>D{gGvkoe0yUi}`E?4#6&RhM9121HB*MVnb-JmhsH{LCr zX~WAEZj_*VRKKm{d4c@O8b*C#EJYBGjXe_yW;`9x&l2{IHS7gv$&(*(=!IfoiYj6H zDL4MagvFTI1=USMEH`G#geE*A21U9!A?C)6l#oo2$`p4xpi1-05=n8pr?%HA3~h1H zZvo1uf=lwY`}{nXF|xYclSg|Oavf$79XTChjq<7V4?%!uoEn;KgfFeWF^hzPuH__j&Ph<)J?FbIBGXxM1jEUZ0Jzm^beGGg>bcdF_^28bfo3?sK$B$ zXc|5I5o^J1x1Q9(*y-iQ_Vn@BS_g5OZV!pG&+ArW*l&<)#x1zop{{9CKYG zse8o;1n_UO_7CKLw)77So_|)aai4!)t#zM&R3(vx}$^q{aE5UN`Z_^qW0d?BcuOy$#Ww*eXH&+vV$|py#9y-hW>f@&^(eQKUPl z<)&+(2cd@_rPFHeWbe=UO?sBF2RV;#3;gSF5E}qK5lUWSKW4-i6(3O-9>z6OPj`M0 z@a%D0jbdL~Y+_oHQ6V_x} zVMeh}MiGQ6PBTjj7D%=VDjarkdeB{aUMJ z1)^7IRA?d7rRo9#S8QgV7@$g%)Zp=lp)7avZ>?CHTtF>7Ntr!j0u{2^M#&&*-TARV z*SdA!v|@m5S1~}>46FuqvZn7qV0Y|)sa4Gkz>0qd?Fq$zub%W(Edo9L{exY;ah1im z5Cw+yJ`_;bR=GzJGwwc2$$UIOmstLs6>ax1Gw!S;u#SCGVEW#=OLM2Ywp@ICHZB0o`;O*HhYH2uU@nJ ztoE!ZL;@f)xt@gPK_Oz=1dfTetTisVs=Uwg2tD?F=a+y?wb!|;^N?DAe69H&n52XV zlTeV|c*8x1&kwcB=~Pr$k0gxh>UX-V6+4~s4|041bY&c|sZdc}U9EchHut$~IM3|E zQ%2Se%{BGgt#%ta1Vs~LXJ=u)YNc0pt*zAF;)QmEL{x@RLfCl29b)V6LSR>Ikgg3| z4qm4bO@u(Bf#Lt> zF=7=16i4o}4UA2Qcns1~5R5$gqWbdT49!UYpnLJL8=NijmJ^6#gy)#-Aq^2=J{_5A zH3?f2NR(%Fb$TSpT9!cQh?gPZEZ)#ckKUjyvK`_E9{ zzJE)NBF=#Q+`wR$ZhZ!buv_ zFQ|rv#%l?RIQM=_N@T(QX08jauIvF1&T7JND4O6t7Q8EGRxAvx|r5f>14DJz1_tx$;Lj2 zhP${rHH+CBbz#g%tp+??kK)_Ek3~l7Vjtqd-|=*i<>}UQur1yFNQ-<%@XW$;a!AXo zCLx)bMCDs1A(@ytMB`q{d9Q1d^@E>~x=|~gCRs=n4Cr2Cju}WV^?J~`J89xj4QO*v zq>qP(*Y@}g>+Q801PA0%rM_4X(pONcA3-(|=-G$hzmTX1UwaMr9AgS+vPtj<1 z5O(ew#bR~Q63$DDbV?||xl+tp{O)2HCj!4@eFyFLZ_antJ?ZU85pT5OfX4Ws;zz?1W=hX335X`hac@jJH zh+GhXFB{!YMytY%yv7!1!(5N{>;GVJsN0Na<^HL@y<+bk`v(!d-F&Gbx&+n{{oCdm zGp3+Ap9QlFSWkA!K@ILjfe&NK{`%u8q>^xeRVRC;n<= zk4blQ>`+jav@{8ZlfW+Qh~gr$<|qc4-9t(iXnKiai0juiIoVHSVzxWa%2P2FzOU(* zFMrk#@p#jW(zpO6TKlElPs(kS&zG9SoKDX^5y5uo8Jn zePF`0)d0doqhO8=hvy#zuhwv`?&AiA1peT6 z9YA@0^t`4;G`Ei^JIT^TB#hUSo~H%Fpb$r9H0y+Bt*h8*!(+G&zZ_!LF811i<}Tx4 znCi})Q=^&jnHdX#*8#O9i#pe=QyNWAm3!-i?8CVr$IhQ851WjaW6~{r+QDKZ#oNL#enm=&zL>B~wJJ57~67I_z}1slEqg4Bwduc6yUYJe8U{1RDsOAZ+q&#}qJlidY?_jSBe{b5o8}cA$N6W*sMe7@gj#2MDxD znQJHRtu7MK6;sVPLkB4^){yA#IF;#QaqhdK+Y>S6b>GEJ_D)U7!{VEIO5_^gzUE=^ zt@8QM<+gDn=xhhunhgv zic?OJ8Atx|4jeH+WotDesOLr&?b3NV9Co=+EONBq#>+)vdtEh70H#glM?9UUynvEa zKXVDUQh-w9gY-N^<_jqnrvB?`Sbh_z+g#eu86aQA+(ohfP%PgSe4h_Xz`|xbB%Yl4 z!-+L?Dkz*)TAdCpw|P!9S;nng8>naH7WJ^zZ^`41PZrCP)DlPUZeJuJ2IrmwZwdhv zLg@)MjX_jS=P)I8Fj~&H7voG!4tk1x_-bpqVzz%z!HMTa@VjbeTV9oPz%cL8DC4=@ z(l|5&Os$f3b1kv-tgi>vW0uC1h0HA8)qm_Kp&ZRg=a+Gc8Qmo|KB?3$J88>tjl670 z{z{0ua_+aH5H{C3x(pddl#CS+qqtTH0RCV*O4j;Cs`1_LBAw0)Mc_=jWZ&gB<%Qfs zQtt7~=Mo91#ecK66OKzS1?ySESW8Ed2+f81;QBXyV(QKGUf5 z$b?M&Eg#a;*Smb#1j1 zFwfS#vK)NZ@}>T1ix+1%v%F0 zceTO+={Xiwy=gMG@Fo00rNvu3Lj0y}-cG-2{a_C|Jqy4ZMps*yp&S<+ZstnKM|(Px z&{V$N-e3!2Q=TxDMhsw_+yvKXS+}YM=51HpHk>X0()x6WQ@6ruDcz(_S|RfcKVE4z z-$*N%?aX*)T)%KR`x|oLw#i;UT%BlI!2Q(h4a24H5j(JFIGnXf^uEmL`#Zqw)%nx# z8}7jC~xeHgyQjy_IbR9)q1W+G_HOHkzHZ))L9vj1`9)dER!?a&Ai&%&V(rBqgc=o ze-?&snEU;`#85R|EHi?iDE$w-8OYs%9h{76PBc^5MAO;aSLHt|Z4Gcj{sslGwTQ zgm;Atqars%^UY4XkJDRIp~k(He*fNUq(+YzEVsBV;!KRF+Dx=rRJx3eF&RsfTq%+$ zn!Ail7nZfs^8|49L@@`&9iYGlF2ic%&koQ6cM)&VxDAO}@8s+2X1D{pPDOu`IWgb< zv9eO(2#ygqzgRw(byNotF>~gvkm+U#X%=?$1pyHDV!QgMtb4-y1Ye2Q#)JXp^YxcEu8&$aq1Al&UDn17p5n|0Q4=a+6s8E-W>jzTD^-8$} z^WY6{Y2zHl z!j7m~j%C1osU1UKIp4XC?yZp{$u2b$d^VW-g@1+oZ|5qVRG1}xIceIkZ`ooKsi zh7)oNvlR?cW?E$Mcz`F_R6eiQ7}B8?Vmh*=c2`WrK){tvlC(w^mo{rJ9-GjduPbLE z_S&(%nZb0ba<AGP+FztJ1;IZckq(BbK?SMcZrSzh{ z7URya&DXRMo(XeoR(KVXZNP#p%bYGBxkd|pQ?bhuHYP_6P?DM-;}qI)kiNcAc+_p_ zRAL359%Cs!)|w^9*`FMxera_y2^Ahh$d@gPug+xh{46>#=QevzZZcAu?&*poz-4$ zc6*-$dyB^H*iDeo<%63l_O}(uvD;_+8jDzU*r^7=;~C@F!BgqAO?02em2DrK0zouK zxND!XVUOd_!^4r+#p*Wd$2B;@j=*_G;>lqiPh~IPF;4nOCY0YCq75wj9FPD4?B&#zt>_Glq|f#^entoEWAmZ&{j#C4>z8Fulc0&T}o1LN(nxD z77^+%M}k;suPQas4EutXVq>P;oZ+W9aIs04a!>#^fXhmD!VFvV`PYVixDY$fIt{ek z!}=8#4*?6JUGpcX&AY^>*FvYN3RXgGoP!%^c3tt9yonx$nJ5wb;EKv}uX5La^kES| zTCaqbr83h>sQwr|b6XNE3Q+0wgHL|+GTmE^BFXd5a@4BviRad+kUqBm+Op)BTLk1j zEy7)>(X%`^%bCasmw&B@sDM|E|AsKBAJFNNG#7F_>L9|_Yp6{UbcQwE*1Y!qX76VCEqgGcza4r#!cISB>yF%Mzt4iXT*$lumz+Nr zwN5|~drki{Sj*71W}j@Zvc4kC9_t0CI$jE1dA0IlRcd3eR zR#%<9YLn^5eqNRVf8X=IAb)?&_ptgAcshMHoMOOM0@n{S~&N|29u+yt_WVm4DxAbwyG7^)fVCv+vEV_A zr}gOic0?j!HHau;#AitbsiEV1Io1`PUk&y2`nCiQRcjOhjD5ft$FgM6&3Wm9Q6Cyk zX^zOqv#M2RJd8cqD1A@^C;F!kZgQ@FpS{yKhq$_8`cj|uIi<{F|6V@F&8N?{6ff@A z+o_GJ8ZYvG%88S$+rg%c>^l9;NA zOLkUV7&8C@qaDp_{9Qk(Cv+icC7@3^{l3fffE^+9LvIU}mn90t9>JfVOo>6WyOZK{ zkBDv&-l-SvW+>)!R=q|;w=WZ=6k|;&oM*(t#sxiDUkL_h<)oy=5>ZT|{; z9O?!4I43m&J&~Tk89xkrjXzcPDMR@*yRJ@M@2Mzy0*fL`$YE^ zgm|i?mRZ`KaRh>Ij!aPNhV~(f_IWob)w!)6Ih&>+1~t-A z__x+@kyT?+s}Opq8VO9k=~8W?na&vUGcUMhMwD z0$}9+SjQ7t9YWwotdgaW=+$RC`E<|f?J}*%`~n^lm{SxzkRcIj4y-qahgWBa<|3IE z^n+%LUDa;EN&)XMHWN@ix;;e;DimnE{0FF1f+pPamiS>)lE!x;%;I| zIhrtmJVzs3_$t#DhSFF(g~e{3-~7kNN4Sl~$bq3;V{DiGA}}a*izdaV%DwItGNxt# zxh$qRi?1IMRgi;wX?@Ope@&@}ysRTI`Xv{}b$|7;>iaJmewv~eqRK*3x1Wk}&Vg8o zN}nYubnmaPw+|Zwc`=6~jlDg0WHJi4X*TQ8Om{!z>=2Pv31^$zoL_w{}EUuwkd#j0TYsI8ndn?BQ&SUN3>s)=DgQGBBfo=4!H-wSm?HzBqPn!3| z^(%3#Jf{cYRAMa?Zk|6U6G>o=((3Q-e6*W~V|^BES&g0fWQbednh^O=VPV1H`x^o6 z##s`Ge?FV{Ub~zD=#2g$PdMdY&Zq108bj6LEDhCFB6@V6%&g1svpjE-mKD>SQUkZW zEJ_aHz706G1Y@6QX#)*ixkr@oh_IYw4o*^dKF*xNOiLDg{*=vgZg~k%1whI%GS2A@ zKfsP(9BC${%Tv(vemdpTO|Vb!)%4v((7JKD1hShi=FEerhc!qmT=fc4>2e5|K2&*U z8LauMx@l(+C_VB#A8gT{)aZmdM4hA?R?;&xRHeO8d(VSuY8kxPjvea68dRwr#}B3L zu3r4cH!x=F^#`NvPWh9d?ar9qW?xfxrC;*KccuRqq4n+%7)Nw|RTa{LPhDx81C%A7 z9P3E4S_=!wiy2TU5I8vI3aQIyS~zwpjDZN>2*r13Joj!aj6Q$iPN-BHM&0c_lzb^} zRi(4kjMsa>ChlEiM;`&4CadEU`z*JHGm5FghNW^&Zw>Q7fyn`L{GlZu<{j{-W-q)OYeGZy1HBV{& zzI=L_DE+RxDI>c{NR#*3b|iDvtF?OsVK%F^TUs@MNsdjE4=}7`(cp+mlQ%I>(TVxb zj$ z6gBi39M*PC&C=7h^O~Eb7=n*iG>$BOmrdi+wazHExuZCg|I4=2vbSfN(_3iAXs?;1 zw+Vwz(AgH6Lb_ha2p|MH&%b_Ja61~yK^TIWZQM!j0+<}Cw| z;uv-Xo~;J7-Le1u*S0%-dYk={7ykc<|H7d1C&7op|14ZbN2Ncb?ti0T>&PhAiey(X ziA(W`VOMC^JSshs%tbERgUat<=KWs3>D&vas6*wUw+jbxU4r}JoT%viQ+43&-~@Pv zV`X40`hc9NQubmy=AO3-v7g|CG5DeaH;3-4v&IT0+G{@a_Q9oF69-mgfxud&{ymKRD}NRSib~g^b&rI^n~? zS4ZoD$oP2*3A+bVu-U4ui{1lk^*D`hsx7T(=xP@P@mXmTZp4yJflEutXe}`<5M|m2 z2wv@`daBijRs`VY2)XL{mmw(N5;Bj#x_(pYx%#X+Vff0un~9)Cv$_$#Z&65gN)WE18bXCJ{i6vo6` z3HV8jFPKRaF{qFzF;HQ)PL{6K?b)fD*RNVGnr%ZOlZ8}9>${S?`Nb~y*60n4d#qMzD`__Y34Ya+# z*lAPZ-n+%5^eAeTqD(;E0n$`cnX8zV{rYGbN{KC$z&$L`M*O;@ z%GYj--sDBXFyvdvKghgzYuvsMXaB69bIIG!Hi#rkCvDR3nufW4kRt2U3J7_omO)k) zUU1U;_njdD*bW=M(P_Dn0>@D1P?BlA|6+>KQN6ppMaA$=m6P^uBposy6EA#>%pVra zb=l?v@16%cz-E_;^|ni%j%T{H#WHxMBQs~CM5rhXR!>KclT6LK$*k6P;#2w=WC5g> zs`zRF{!(Rhk$hj5jDg?F5|DXk5+kTif(<~M^v^U#wt&q(mUTJSUzy<>_in}MZO@sD z^rzSZcgawX(>87E^+_@WWmceewm=eyLOouK$M>K3Ix{I9bUVC(1Npa`b%NP*) zB;2m(Jo3nf--7<RP{*GZXDGM)=TvXoL=oQRhfBYNFl7@>22bpFW z>e4kqhjE!i>kqVdiM{qyez&gJb{53cQ=HrpZHb1nA+)rr*qX-5yKIuy*@aB$ji8GJ zGT68s21YT97SBfBbU6I8ZQvr|AR7DB(aO=v%=7SR#4_oIm&){`ZtP8o2;YWNrgf-% zQcr-xVV8RUj@RNAc7u=gzU=na#m*Pk7#Q7*OfZRU9qy7q&ptT3;&{C|xMF(tt0d-p z`4EzIpw*(n!I>D>38V*sgcHr9FyFxV>-=zd{l#OF7%6N+Z<6S;TmT)4)b2&~;$^At zN5eVeSe6FeC*EwACroI%WheR*_ z*Y1MfIver!GHNwMmQZOEMVbVI&3X^7OT;^yN^TxWQ7G_QXuD8^-;`~k(n)7!`Bfsd zI*#NyR=q-#A8~+rN_jArmwy_|Q0|&sr zqvSa`?yeKOns-EbV;SXX-6Gi5Y26~&^l25?L9$)KacZ~H*RfZ$x3@9ahHVk3YY)b3 z+2(26@~KlN=whdx$=O0`S+lv~aII^<-zaJ9ED;tKL!|Zt?y%%~fF6z@4`-p8B^k@D zXvl-mFXmR(im5a8*GIS{9CH5x+I?e`fBaM6@%!?9A^7upJI}egALN7!D|QkbWIdHj zW;(9wg1{P-kp9H4fMUR89;l>+?!tGZlBfTcz^$k$2o4sBKsrEfvwgs@$=PQq!9wZ^ z7udhr;}WdyX|ms}93=o2sWs#f53C1@=* zn=X$THcA~QT$7hl9->h(EpMBz3hQ$J3nZpl6Zm{IcT!Q(O1vQx3Ae7*ru?k{1cUTN z$kYNGXaHhBoxdi%AXu)E8Ho+Zv{mf@E75p&u(}H{NUxH2k=;oGF}PXlL4hl@3`D_2 zZ^VXSIJ7*sMl{q5&iCt{;P3sk)o)H;UqQS?;P*SC?=Rj6;cwUP><{DZ2;s}?>r24L z&v(Zujhxr|)%ZIBG<>%+VA2M4Xj!ubOTqhRx!hnPf8u6eD(AIIk z9?z(O3B6)8!C@t05vhX+dOY7qNa%3)L#V6mj<(1xO&>@^OHbngLKzq``L4JwPIb{y!t*f*>ce2ErT=g8p zLAxGm(wXNLuIAfz4$T)&24)(&6F7c03%5LlM@^5kgab!Od&LeS-&2i?!+^ zY|Kw8UFO4d=>4R5%A@t)RxUeSnY%W?IC;(Cz}; zGO1MuGJE|d9UtyB5w9E)*#x)ZzRr*z<+XPRG!Ra=UHx{~^Rusx3-Iz=1Fbp#I{qiVwbP!Pk{iX%uZ?ftkM} zd3zo3WPA9F_m0Z8L^?e(KV5JTqd-g*YfSk{hMju(0>~$$)ka{}!-4{gd-f50ieA_A zGndz;z_HQfXWr`H!1LlvQIVD>qk$EXO!?IV4**Me+Po{LssLlVXE2l#TX}I3kd>b3 z74_|o?0&PK0P4ez6Cs{AYGF@C$@9)OQBJ#a2(0$l7G$cJzaXQrdd3Ox^!IhJ-pi_=hV>na>GRm}rV+Ep;VIdHUu4JY(}^F4%C^x@$`!6g#c`)} zuMNZJlg11YdRLklJ-;}k&_v*C+rfJcabj!(GOQ z&3%!juT6jqp{xd@poWtwDiPZu;_5g`4ijI0!EH|ExWhQtfqC;Iyn5qbKX*e#FeIVN=;W!Qu|7_`S1ukZbQy4v}G7(A|)Xh zh1NgeRL7B3O0(nYTs5l5X`rD_gYG0jVSw%+uM^kPZ;(sVNe*gq=g@UC-*I7eb#Jx( zE%BJ=3JMLw#Yd3_8o!bEhvm01tYG|V*;ui?>aQg2JDk`Ne5ctT>|3BexFQ6IX) zqg3ev2A6q&Ns(bzP)?U2xdy4{DtX(Qfzk@JawMK0XKFJ>YU?H(CWg?HA zbwbo0nHzmADeVm(h5MvIKc0G+YvP@tIqb4#gxZLT1<+!yL6D4J_uAcOm(K7rB zHcgz)QA?3%o1K1@6_=BG($gf2047Ynuh0wSAxLmBt;(Y`;t4 z$y2zPhHgM(0Qsua2t{iJ6^nv+i^|TS5p>3S1ff-YF2K*05O!~u7D>@%H^ z$M)t!y?M|$)*ScUdcx;(9y2$?=?A5>Z|wQtC!EAw7Iui$Q`?777VRARU7OjWsHBs3?Wb z101pclt{K85UBi@bG#E?jz!m%Z2{HHHi|Tc?(d4XB>uQVs2wsksoAEeP4=M9#-gPh zV*PzXT~*d0<(z}&JFLM6UO{OxmgFscELypH1)rpZGBA=a?98tni=Q{=m=aPU7fdvo zcqD#k(}hRgZM?u|je9|}SLKqF*dK-ZbrrZ?HLhB+>!s?etphf^X*Ji=&Om&dV_%X(6J3A#jGda8@G#?a} z%V3!bhEX4Gg+1nen@+mqi{ex=Nap0QED4AV@%#W;1eQpZk1L5+=bq2l1(AsllB{BGHGZjyz79$zuC0lRWT96vda0SN`U4q1UIuBz zIN*=n_Xy0=YgpX&#E^7Nnqi`N#Frkcu8B8|aUJgms4-Qg=ig?aLzR9a zZ(i}NiZRB{(pGai5BxCZ+PS2-8&bj zjdwk>G!-nA`)RvE$|Cw03j&r7N^M(nZ*abibCwp?0a0fw#vR4e{qEoeCRfqH82P6k z31l}IDW<-e4vAN6%*N7>^|dBg%0z`4_-XSYYF4J*nKkU!4N*?HFjMj_J*kGA@N3Fy`tIYhFt`PIg$JBsx|D%v8LN;paT(}|1{n+Mm~@H*QPg(Hom}JK zO;6tm@C+XCJcg1pj`|GxCNho6Z_5at9^UuKR6P&pfq13%sqaDmslH>u7<`@C-ZS2n zAX8G4B|0ev=o&puIa>jzY<-UMA_B$6y7vf;Y0$z(C{KxR64*VEDU^V69RtHR1^hhH z2TrHBKL}TOxrccB(BF#XiqgC0XrMn|U1A^~bZ{Ue3v{ipEFJ117G)EIwlvS-PlED2 zLqyQ$&5}DChSA=zx&C9N+<>)eJ*z@oYxwpGH-S$tKLk}TpDLIXr~55NdJmRO=L~@V z$A@jtP+Oa|wi+vNf3!A!K@uO?*#+i$cwXYu^b+!;t0RW-(5go*87{YPH1hO^bcv8F zmEja8U2e6)oE+G0osPo@%C7SEnEK-75+wXqH4G2uAMlh&y`7Jt>}U`^Y*h3P?hC-J zO2<>~RdRGLT>wRipIt0;D-8FXssx9L7l}V~5X3Eg2Mwg$MKb3D4}eSHkTL#;OFv=R zA+nBQa4|=DW=DkM*DBiXXJJO5^PQvh{Pa%SE4UWf2nD--OmkO=X-tY`Ki6O%Ok=nmMF)so*QB3JD_Ecv>tWmD_iS- z_Iv-+iMRXd{RS^7)`KRCDETGF;4PkD9`86?ME6O4QB4`@%Imp1A!2_}>#dP#zHGgo zn#-JUVQK>@$C`xh!pgDVcx}YyIZ&@`??ZC2{bv`+ImfbjrW7%sIwuDAD%2|`sfSy@ zmFc{2B!^U!2IN@%>0k?-ExpvJ-EeCB$g7P#s2`T@@uuLe;T@}Jo~)WosmkWP!KI#_ z=ITRSK70vu;?OSx5x(cc@SS%c+%MZiL~$Mz}l; zjGwg?P2BIfSW0&=4e4)#T$16Ytpcu7zFYwZ=8+-JY{9|h-u(OZf~CwFjH z2k<3a&IVezt>v4heNW~vhqSrQjwb%K9uDcSGgim3GtMawBhpCjIdn!c7C1h@d$+*oya8#XvMcao{*Nv_ONx#gtM3f;yj5(L#hm?aDqM;P+vzzYylhmEY%?17? zqTMH4^CvvuJmkLebm{*qspBP*n-F zyPtJQ8=rC9YhAlfi&TRM)hq~(u@7DfZqHl}?{){Db*ixU3i1%;tc8zTnInfgeVCS!;4P52ln=tQvG;o#J!{D)h{OTP)*aka^EPz;$uss! zxN^Y7AOm+a`~aEfgVmRNlaA?!St?!#nEjv}`TLd4IlPMJa)YUTBC^JWE|fI3QGy#R zHGHOGNhSYC)Qp25p`?RPH#yFbK;#@d zp+y;rOa2T=Se83h_f-G8O!rOfcy8HQ<7UR?a)Am?Na#(V#}5Q{6;uW`TSZh%zuAbl zji%=wmu_7{XWD`t?6t4@@4i5^yL8g_jPo-jy@{mgjt--MN$&#F@c{KxmA;m~7BJN# zD_F_Ho<8y-s@t=VuEK42ZfmFCkES>oMNFd|GWF~&s^Xg|7LFiP{xJr%b{{+qn+zq% z9g{p-*j312Y^t7u#yORiI?0jK>Lzwu*);O$m_{U2CK^_)6q9SHiG|>6kVQjJV3)3| z&b^>PPZCuEasP2y7o1q;X3Yz!9qw@TBK6zIHEhb%oFYfw=^lY@qBH&PV_f%SfmI9a zEWwJC!GvXFq1!KLM2<~=%(2X@J7acI_Ywjj30el*>tGRULXBO+3Um_L$bpOEl-p+z z@4Nm&AQo=-I`EMaSQ|}}=2kamyShIzej(HGi@%~zQ#(sjWn}b-HUuxMDpjt+JtU~z zXbH$n`s@hk1?!06L|3}Wt#BlND0Zn|A_+C9>IBL>Y8$Y-qHyh}dRG^=x1$hoM&4vk zHWWu?<|$1mr=VZxX5-4fP_ccDW}rI%RRe!36Fj>3E&!&xAH%5Xc?2AzypoPzrL3&6 zn_7|`%K+PL0mUoV=j(duez%6Z#04_-B>|a2oYHNy!{&sg9Vp>nS1~A);_|kA)aMNwyJ$Ej8>{T4 zRK(f+)fC_pdkx^7AQ4KK|kp(Z<4+&PC<{HTK;eWy)x3TcN zrbsY-i7W+#D?q(gQ0?UqwPcM zJ}>eAyD?-xxBG{l#zqiC*$C?Wan;!a!qUU9!4Q812kp2qZr;0Fgfn?t+OtM-;JhXM z<4n8G%-U*-9zqwZ6B@V#3%_ljdw3{qt06(r^b2qX=P`1gZuHY zHdOGLG%#$y>WUHbXa5krX-@Xi2G~lt*>Dd7K)NKj49rY=Ju*pIeSs?vH8@EiGJ<5- zgy<%~ErZ=4XRg|){B>{*$1Uvb755o%Ly=*IXb4O`gvNI=jAiDP4(tx>XgEI?h;C|{ zIK8b!X|DGM02=SPrR}SvVg1AKLm0VEuJvU&WzjhKePOHzk;GN+T;vdi)^@(gckjLV zPakP9 zgjHPbNz~67Fa3OK~h`%fZ z7W^H@qjI6&MJ^Z&RIe!R{z1r+OV!|f{)gQ50DW{;H-r(b_+(mx3U)Vd8MpqUQ7^)IFA@k07RJ8j5X zp+bg=&Go&cS9B?V7Z?d)bB3I1%pfEZrbAw~ygkiL8g)6F?G)S%q>Wply0C4oZJF=0 z%x493)bW&=w6Xn<@!e&Y^y-j2(44j!fL_la?b?9sUn{~kLe*;rgTpo1km_GiASW@* zDcUmbk&ff~vh(nUa?Z6&DYGZ5t^=ECHAwbIecDMbXu4`y?w)hVeMz1m;TiNf$657T z;0D=qrG1E>RM18?RCjd|Ky3q>dwu;~bqIhgn&C=F?|B9vi>a=^I zw~lF+`$jnYN=(@4WWzq47AWI|8>N>rLyP!#LV=hH;$8jNBf61ITs>>e6)#%xW`9n+ z9_RQp6A>x@`NkXshMBdJoHbUn%2ls+20yE}@@I*nitY%BteY1sthTgcUoQtm0K&)(BC<@ zLuY$+h>X_0DNDlGz!;UXw@*={%j`@04$ms)*aNCd-4w``?oZZzY-aoZK)fN_(BoZ& z`uA8%?h_e8(_~!}Me*o{!5uak!fQ>A0MD_FrjG|bq{&qNv#cpFCjeW;r-SUqQnipP zXK|)S3iI(F*F4Wog$4U|h{THw!lY=txIDp9zyA zU(G%9ptbd$JBEZ_qX2dahcanu=K^P58|pmbNuFCx6_e%-Ps13pS~!}Ng-tFS8&$|AJni|F(_rXFmg$Pn4_gjmM&(?^iie#LT9pAo-BfK*Hk4-(VRGH-Stut zW}FK(aP5opc&rsaRjjhCgsrac=RJaP%j4M>p&70&lWd!X($p<5oau4le7+<^*_D6& zQ!W2=s}e?DMWlGiFz`BG=c838%L+0W03bi$Lsk+mnOAs5Y`gUSwgPd$wK^L||A&)+B|%k00Ss&hIA7sNDFhH(}|ynHG~ zyF3JhY$}-**7tUI-SGK2f`nv0%Zj`6q}!h(@hJ8~C3-k5{~ovFC-V#Dcmym3}OSnb-O{_;W2mkWHPn{Cq0 z+17n@_n{VOP$p`xnByD5_Md}xGmmG2bhEXho@d$A8+JFyd?crv2nJiru=OQuDaS-b z74WRb1;j%6I zi?gcQd)%Klv^SdR#THf}-+Ub;_Js{+xX2hJy+daJUgi7gYi&hif>+AJG#QzoVnzE) za?O}>b**VuS>*lZBrHXr?!Ej^q5SUz$%^J}brBZhb0Hh+3|~l22?G7D&w%n4K>=Fw zp{wyKONJk;5RX9GRIM@IcdR(iX=U6{yRQid3TA}0*9=QQ*tO1;kEkptlee1TVa+-t z`v#X6_(HY`k62f|J0K_tmKAjFUkgBKPNAUYMHh}_m6!g@PUWM+H>$)GIT-6MG=hnT zTcXR$v)|umP&x#RL8E?;xucl+?=g9~Hj&sv`5b!M^_!_u5YeATyO1VO)3 z=v%J$q51u&+UbcCt^MP1A7C?pIg;5`Qtg0-?l@U0agJ|aTb;d{BhAO)<#Araq0tK# z0G0X7qQ@=VzNt&LOdCe|?*z7)^@?xI4{W1JAO^UqJMDF*YJvJf%uIuLbRoxp(OKBc z$$P}A@dyNhd2E2NmU^=R;|DJFCFm;Io~m0hb0Hwf|xwGybpQXBu)3$kpg7ZKM_k0G2*lg3mwY?S10qNa7&nI5M9NAeQgZZNcu@NXt6bfOr(h6 z#gDfaG``$7n|!dYFrhBN(KRpi7T=j5Q9F2-xPa46)+_E&I5TSWn12E~mI>fMEt~?P zhxoa20sdjeufk@KjL}_Ki$VExnWG}NrX>|Jt3nC~RHB=DQbjL?z5=M;C;ANkhwNra zyLN0MG5z?)eOnj2;kC_;p!JE=ocgFl*EWtzag5h%JIS)s*HIqJ*8xOIngU#Jc#7X} z5%3P`C{&*E&U*xDUb29J{H9d|Dr>_BVYx^_7B-8Woqw_NwxgoUWEZ(6mvvD4v(i89 zy3Tk>7bxZF7K6UeeoMa4{KxMkhKd-y9jVDdOTSoMzaBvkr0eRh^i4jbA=}=oQw2Dn z50^$1)I0Xa`|dpb|3Hq5kgzb42CgvNZSwzKPV!pAwExr(FXz7ll%TrA{-iTPm>mX$2b&61H^R~6bjhyX`& zkqx(8JE2Y-%B~q+MS0)+PS=%ndfg}*nPx1)o$Hf2?hfnhN!bb7VALwGda=;drKw6M zs}fsAjympbwxIAbkOw;?v1cKY6AdTr28}$wgMkKFGlYh%uRX#E{;c`}K;Pzy21J+U zU3d@c;bNYdf@Eebo)((x2`152^!zNP2Shx1?R=$d->C}~v{w4lfl{P1-u^Z*fU;@T znAr(S^4ikgpZ?79mqpjEii8v;&^ddh`=%1&QKg|Wwy$$$op{Z)eu8P19yD|QQ+9lp z4HPItZyq82&fuutJi{@P^#MHr2?Rv+UUC>C!;BDK*e$TpSNEt5BpqzQ_* zucw={ai_bBc_*bPpr)LGxQKdv3+-0`3~8A8Td@b*Ajfa~91rRt~!G z&c2N`Cj|fN_>FF69Eh7pIJ1wH;fqf*NFU9jFeY`f8Z5ihKi)Y%!qlC0u>mI&EmnnG z{I?@oZa0^RwoI1SgknbCfkmv6LVV#)KBf*4!)g>vRr*L3*pL;5_}m2Qy~{v}PMwdc z%M!}*0??Gz;~URoMZH>no9BK=C{e3%uu@eD9ZjW(SIA+AzR$xvTvFILfX6gIL27Nd zIoT#^ond<7=GIoBYHecbwOg6Z=DTaEP=hn=|7WwYh4G-Dv*R;F)QY*tNYobUOt03P z46Qj0=d?pBZXr{=b-OCGa6I^R3B;YBBtdxcsTY{uD}yT?T8_I68^Q-U$JHZFi_pj8 z5vVFr`_qYNqGX)b4U;5ii6Al|Vfj6->FEy|>vviC+zpF-c>a)pkhLd?N1WYL003-8 ziCAxo#NC)NtT!ge91E$K%22^K@rF8?F{!Ey)V%urgvOUZi3)}oq3>f#|Fn{q0bb=cP^+jeoo6w*nN=x70*P4kZAAezdQ{5k4`d$! z-Ph@akURj&Z%gZ~mdP{=sb_zct4E`fv{oZbiw`kqLTV0Zkp(ibK?c`ARG3-vDRNnV z2uzYogJpjpF>Mf1xuu$48?}u1BBq2TX?73%B4$tpwXBE{jY_x#GLcxs@KR_ng-sL= zwLD2lq!H-u#z}R`sEtTL(IFT4_%iNPAZUlm0B_ZN3tk7MIY7ikLeZDvU^e4{9i)uE zWe#$Rr>XnW0M{!ctjra3rYhg@+ytB56tk1~c!AK2=+B%h3ltHoFsCzP3X=zP**+9z z(7|+n8&=?%0k9q|2%-bk2h?wvNZ+T`E7F&r(Vp!2R-_L@Ac53&tiPx?)(-ZUUqdTr ztan(vcNi-I0tEq{^-?jeIqDQSY@d=XZEDxbSZl?&@ zEZgTbFs=)3j*T13S)}MjpUj6E4(4czhSuhe%mpD)_3gBY^!hNxnmpNf`8|(-BVgz) zjRCTTMpjhZX(JC57HF-DfiTRi9bJ9rK1INEqc7P3s8woxd z#tN5*7ZOrf-;2KRIYc`$T}VKu7w32Z!d@3ub2Y80<~J^Hpx-mmlrmiIi)vSuabgFg zMH)+$B$!+*$ON}s-tsT0zDV~U7Y3EvO?H||MBJT-Hc?Qa5d zsO2i7LG$lzMI;n%p)sVN<=a6HtLJ*`{Kj!kHHytBzZ^Lp7 zmQV3m>k_^r`(Sa|vKRuf#GOY_^GX7u_c|$Y48{`HeY#PcZsSP|Wks#Eb8f35KEn6% zh@|Qz&RSs~l;S14{W*$V*f4Ae40}AoDsQI(i%Q8xo#5ls;WP}InHOdxew`nt{N5~u z1^1b;9f$Cgc`Lq-p%l)%=3TEA9oR8h1tP!L8Vmh|m7_ni9d9y9QBvmEQ-VLzo2l~I z0K7p49E!QZ(X%aVespZZf}F9-bXL2hgf)>al)thTkT5k@Z~>^Ae*VEeOOWBu`iY?1 z_(>2)*Ny-spzBB(_iCQ8AEC*Rf_0w}(mqfr}l35yDuqX^hL1q z4PLY*Q@Bp&i(7zCvh$|w*{ac&I~ljY2se!`RJfwxBu6e0i3fwz#a% zb`N2aIZQfEzx<$MbjnF+(_6ceC0nK;JJ|(h&p~M37lwYn2$*DaiS*9URLZk{y9zTp zhcIcEjKgi3Mre=@bLgEu2{*ZM%cJN?IbMe9RYb?WmlQh;e+_YQ6l6|et`&o^OVwD| zv7R3Rb>y=BlrHCdHv_t*7&o)gkS&9&Ie#@|N4b&^u;J(mC-1KirI?DwuH{#G;xFY0 zbUd~D^L)jT7*zfoleGByZXmKMzMtLHPfUY!~ z(6RFz8tDfcGs}L{KZS)AM^9fL4+Izlv$ORi;E|XmtHj_`9N(FYkcQFWMhNNF#Z)89 zR<)l2I_>dEORp((e#5{}M!9*lK|MqiYP&^8RWa3y2NP{cAMi5tTB=&CGyb>+F=7S@ z{O=wwr%zuW*Ebs%Cu+F%z)~X!&XPDm&)%KXmcsNAX*4EnE@@q9!CiR>T!ci2i)4Zp zw=SU%tcn?VM0o)}>w#@Tpb zyKXOsi}xLUN_L+#%3-z+$1so~866yj9t<#~Tqh^P+0m27AuEvVw#dguYx$Fq>rE6c zlJp_l!<6g5N66WUI==K|Ctq6zxgLFc5C*f1$0qCGK`r3I&cip<XXi&{IW-%$jGy#! z+kkG9y;t9!iP$OTk59FGDa*5ckD}(PsENXhY;r`fmear7k!n4+)t10@lSA?@Ajtmw z?YceOk;R*iAqv}`_m=4Mw?kMuSQ7%nj=%F0q#hXgu$ApDPWJhr9TBO=0eX|Zwn@?N zJ_%*q1y5k(k=GKLnr(cKo17ETAX_6Ia;G}9-{MJg<2qBWB$e&`$`y=8;EeU}@~0!4 zsH=PImjT0P5?9k6YW72j(GaR#`PWv}H_LXG?BML%1Ud;LHnug0xG~gUK}ZuD!c%rm zuS^b-QWz=^dA#p+hHYPn4nasdqYXuOvUi~Jy@{$t?7{i9A(meS*P)Q00v&UO=uLg} zG!fsb8S^TV!(Vd(U(D3Wly~3Gu>N0XKOCQZ?FjzsdwW}-7oV?tdips(GOzyck!X)R zj~D4Z`QsOJfw|Xr{>^!L9hEEd*rry)BjW)-Y$Y`C=OqlzmOU$XQ>A~e$yOgq&XM9i z5V;o(<_ zS^LcSWa(~c5o;7BALk8)fv@$=u=3+H;cEB)t`|)?u`WaPN5-ybeHp&2*!2;^A=o$2 z;U&THoYxIRDdOOnV--y+SnUX1LC%^{N6_V&liZ{83v5bF5LiwgE3&t)pu#Shzy+oh zdL~k3xog1p^c!5qWE~n%c?vf&oFag2G^*HTGD?Tu^+)6;q%@q~LofL)`=&8=tpKxf zo5Dz6D`;V+V+`TJU41tCus%t)giSGE~lzSbybXbEu}JwP@u6?T-CV@$(bSDmoHglmzbwCQ}Xa; zDQ=m%cVoZ}IDOl#7XOMIB1R5j3lclN^&mn-ql8I4Gl1U-B=K(ZxJhF}FM(qev_es5 zS<7{>-@nwHc$_1;Rlm(GiHyLt3A1pojFFMRBx$~b&M*xrSEN?vJBQkJ2!1R=Zz@9g zlj%T+sAE`Yb@(HjKE)Mqkj}k1*?pfWyOcI8j-OYNBwOFfUdmn@YMry!k-b~&4I#i% zrc8u78Yt=GE;H-4^l3jIvf&YO3)QpKM5ss@Lgv+~{!wA0&$se6D2F=Mf z8nW?)Fmn@4Fk4D%a1y$p9d?$&K$5SJ(5EXUc2psv0kGt6 zNvSi6GG|tY?*i7QB&-WL*4X>df94Vegh=?Bc)z;8pnA_yg9b6(oaB@EZwu?Fx+o&W z>BR)XP|^oQ^&|Mf%_DYNDMnjdlLEy?P?q>|nASZXnTlcSn&5wFZBVu`Q|2Z!dY|)gIJwmvuc5q$wTcEHbcjWgmR` zBN>;{Pt`h1&>Om#ThtKYX#bkN10tf*6wmKCgJ#Vbv%p5m4voEq&~_YpPzki{DTY$j zL}q|B45OT)ReX?lzGubLQ$E@0eo5mC+GX8A6%QOP}>5Rxw=#U(&-R0+H)g2S_(`Gjex>< zv^|W)O7CIO&9)_6e3P46Ot4&3@CGma@Z9;FB+KFI!YuZ`2cCpF(x&s;+Q!>)3u(GJ zKX7A}n>YB3+gEWO{@_N&-Phy1KUW>agzZfhrsfxC;TiF|uHCt9z^RFkg?U9Gu_pDGOU;jIl@!!Q24R1z~s*w$j=rC9$9HC_pAm$41H55^z{jSFWhG$ zICXj`Ekg9x3mrhkPTV>s2`<+OIXH~S!XMQdpZq^T)@I5pY--fZ%fCr&j5HxKCHLe*6@i|Hm)020)( z*nM2vXsk3WBD_*C+3M=;=l=z*5zkd*fq)y8|No)ZLodRqG)GteFInsQ<=TI03PbeI z6hf62fkK+*T%%BBSm%~7W{Ye=>bgnZwI-_iKi5v@w-Dwl)~ReDrrBbitfV$%O%!l5 z4~W~tv!X90rF;;g@LseS$5Q{~n)t?PjXJHFqBNpnwCS)`!%N02{CpfwRZY9iY8ih9 znk3BRswW~+!VP{Yor-P$468#f#xghwjpMHRD2R1jSfZHql2S8Sok4e+=)9Z0iqoYD zN`R)Z0KZ3#^QkVn_yDbl;JUD$=0oQorl1IQl%azuQ~eIl(`71=xoU&D;ZXak zO)z!VlrO>n` z0&GK9pi$r~4H~rk<)X4_LA(Ve*XU>}10#~43O={pkJi|V>hQFwX-_R|8fvvDxmkjbs8tYI1$ZMbh!ncG0tmydbl6t`pa+iDQ+;_* z#OAzL-Nt`!*SsgsyECqeX$xoX|L?*Y$ka8Ol&GY+gjg%lrw{v6^X-zazEiJWf=S(iA2S2(-ad0_EOze;EX37|M7` z;aM*Uf?209I|Mv3(+z2>j8VPKB@1PkhPS4=s*H1zMMsSvB*PEF;=)8MsY}{;pSmuqxdz$C+73tKK9?*->Gc6ZlhY0BX&>-! zvdzv6JDq7jH_qNa`3QMbLyLH30Ldp>U7w0S=mpDDc72?V_+0hcAK91``68w!)>SK>gw1SYAPM`r>g0iPw3I7HIQc14aSVKQ2zwY&e?fS9OwM7i;oiRsExD-%Z7ez zgZ$?pOMzdGarV#8(^UMipAYM^)Vw&;ieV978V6x|{9v7qDOGOqRFx@xG%*RQ6n=hS zxS$qt(KNbPQXLqCYw$jFW_9Ckt6R-ftdtxX`u0+YA_JMDnx+IC2l}K2yF~c4qG{6H zOH=D@cp!_?Vkuk3oyU;qkXQ1Kftv9d9ab={2jTV(7lZ@W<~(YqEW8xg+s-+q%8CTiRzv^%3Q~x9^Rsp{-tK+uV`w&GOk;fM!iQ)xdr!USOkIU- zuB<$DQ;Y@tqljPxwSfw)piz4@%9ICro4z{*oE}awfXKmO@Z-9-gurNjt7k<;D|Z(ZO(sb$3IB2vW4wEw>mbyN_~2|QhP;`2OX5s~ zSnp{(i!E&4jVtBsQ&vK}2enGt9;Y*Kr3o5MU0dH6XPa;@N8i`S{Y$_6Zqv{&Dp`|_ zK930Y4g7u_mH~3G!r;jt+B$se(bsMJ(!f4X4m(=1k8@&lmqw2Y zC1#d>o9HH)HUNQA`YqGR*YiTaF(7V!Z=UZ1XPk>~gl7;BU$|6^&Np8d_Rszv8}_`t zB_?H@U8QNpL=Ukri!8dGcBMcGgx4h=yrN*f?By)H_O|i)7Z3ePM~~jK63mH)bH8Zo z=f>Kjcg5n7vTi|2@LrhXK=wl3Q1|!Jn*9}CTdVF?QxVc1z3qoTsn=9o@?-t{PcbFt zd1)E-(OsB0ZKKw72lE%&DF&y-8~MVh-zi zy6Xc#j0G}Jm#YN3I=JTx1r6gXl6|h_lcje$`8wA&K)giH&ekdwy7QiO!kt7*IPc1c zybkUYxx8$Ag>e~M!p!+ICIsEtj6V>*1 z|AA`vHjJ}ew4JZw|D&w^t65^~e%eGie%s3bOBu9X9_{THX8dbKIsCVZvi%!oDLda^ zn?M$3`bxjaJxk9|_m=;+(ysfTj?3fg4F@6Kvl68c={HBmK9+2n&@5ki+kSRSeG%^N z^R+JnLSrJ}!PGQMwcb?UZ&s!(x)G0aMnP?TX5VM3H-2Rus!wXLqep3}%pP66}&)@L|HI3viDRq5Q@1pE?b{ zU5H(ffGikxH-XIbE|!X5mWKvT>0GP`uJxn!o0W}nrW)aQ=d9!+N6@<<;3G0jmF z63J!Hwg9S3%kz$zv(P~R7FIFK5%?^YiVN7^=9Tm<+dE-wAhSa;C0K6*-YKXh^&plY zNM)l+z*SJWy4RSM{HsE^!PQ(J)-JS*4C3tehiKR`JZGzWm|t0Ij#a; zTW7Uc6!#KJJ4hmAbyCqjSXt)oLx9lg(C)_y@c?uoBZ^JHyQ_lL8^80V!`(}PCj7Im zIAT6$`OQLR=fU#(v>qt_=O4U%M7aGw-}-T>#6dk`zdf&W0$N2q5+x& zaTV~SVCwRdNAIWZI*wpr)U%*sO6N`Sq?The?O{h_V}9Rv`jY-OL7xC0AJ_X9hkb8v zug|l)-^bLRNb26ceqYX4Y|n?c2}yam|9kp?;m!~uM<|;Qj5pTsP{w2SQhdS6f{vQT z5N9KEP+u>j-Pt`^*q!@Q^18CQaY|mq8;8&^JFZK3Os%6)M2u?6sbgc+=YH0BPP*6> zixVW=BY{>{ET2vS)nDlvzgy!^>rjgZ>kFGWjmo_H?(u>aPTZsp6k?$Ro#kQ~1zd9i zlu;#~1pO1uSSCPB^>E|z6TQu})cGbBtf0^fQ@0Zs_^NiE8Y_&lm~NZ#uM zFumuvs7SL7j(SgGVGHwfTw1$3-waF9M%R>e+2|VbYSQDF$FSt^LbtU4!35!OP zJ2)TUj+et4t9+E+0D$4Jw|~+nL|Pk}R_fKtTqV+0kVswH$qSz>#%hBeATs1;D9FUc z(iAAXq#VBDoSTfTS)vO@zP5i?M?(xVeQiy2ZNM}|7R)Q; z9P-hzGBVlVv`w>LW$2p z0f62keQNBE0AxU$zvY1gmP-!-yvEiqQWUHpk_QUtZI5{`hEpliEgc&%=O(vCPbk4y zma-jxYBG_GV{J8xSvDmc!Bow>bC$5)_mn25%{tAtI_>#wr~?1e{t~V42*}81)i8I~ zdD7Cj7l|MG?DRwfU*xBD$iimV_q-o05UgtFBf6epG@jtZq~z<>{i{ViEVTX6vW6zuEL5#R}6bhiVu} zu*HrYXR=Ef*5X@`Qqdc>dnr3wy}&?tEpWwMb}-8+GD(s2++{_5$)e1Jf(nd_4@^=k#4bdTafc6eL z5}z91OG{QF#D|_govhs3IgpwRSSU1=$c+5FR%wBj*S+9!7R1A= zx(Pl{d7V#zgERVJr-UOBo}wwZu$FkD+_X&cUIa;Bj6@$5&kNbi{SKo_n8qiZWC!SKD&duPWSPYK|MW zq?xjUt~KYKg1N-FRyb%!5QvLnePLlR3w99eXSCWAQ0Lp3jG=*bM@jo2C^8lr*t7g7 znr?Wgrq=MkFBOI#gIiNs+2j@Wsa*KN;Q~pp4D3g8B$^@n!aypi9qankny|Zn%c8>d z^}9j!)HXuE6|3m>&u)x7#5Fy70hag&kx@|^6vuEo$XS*bt-K4S1}>}8kIzbJ*?7$- z32=M7UAD79+u__vDbcxQ(h<t<*6M^Mqz#N3 z)_7oJ1i0H$@u8BsHT^OXI4`oTCYhQd#e^UR40H<`GtQl6A!&Jt(vlhTsZF0u`-=gd z-1Sj-hp!k66Y0mpG5=88zoQ;tnPvJ-*PE2<5Ju$b)Xke00Fq_UQhwO%jw+~-@4`HRXZH`{kq z@EakY3emBv*B6*3wS%PuzRzLl5g$o0V3)97CHe$TDZoaXi4_zPr@qe$#)!<0I<$fS z`cWMD1J(j}nvfdq&_{f&{S3tp+EB6xv2>fJ!!C>;=GzwxkuPMSEYeVRDDFr1WoU@eKoThH4Cv>J}xCTyM->0c2ok}&6#=AKiww|$#xn-sr z=~qvuTv)~W%N&=G->arhe)@uF{$vP!>NW6*bU64{zli4xKz!qVwFz0Ffdz{W^?h)8 z+85$RX@Zawa@G`_n2DiIF}CjIP2nt$#08q^md6=bo3oYa%DTO{ae4g91?=gD2Ue=s zFyVWW>Y(J$twC+;b^Lt`7-I_R^6d{NTjA_;*rMU>$Zm`F0OXFBeqDbJr=jo%?EV93 zy9PZS&Z$a@ZiRP6$TD3(f2ThgN-U|rZlmeuV3HmE{{Ry}?7w;#Q5-&hSLe+h;BK~wlnlNnG|jozV7y|>jg&W|&7KTa(VJ_& z)@PRN=Vp?pgp16_#e#M!N^LDOQIjBvp9SoPtp#Nx6?UhI)t}6c9_&up-K}vniFWyc z?s4wI*6+6_D2yHs${R(Zgz0He-jS&1wrP?wOi^Hv)kz-^nK?f|kJgYuh%Lg1Z$D zX$>YFkyHn@3X^!t@I7sBIvS|)R%cY9uhU05umvYJ09Ag>-_UyC3`m+T=HsdA9`FewNNhsj` z5$8e7Bu$}CFr+0?m7CjzCL3@^QbE#6LCvF*(R4f4B`uA(%hZU@>!$luf9|RN+{p7( zf9|RN+-DKX`BZ-{Sc|9na|1(ssz3Kse~vuWpX+{}>d!sZpBoo}p6bu_JWuuKVg~Y5 ze{Srxr}}eG_2)*n|Ed05&+}A&uJ?JWKiBd61}ZE()t?KVr}}eG_2-`I&+WYZpITOW zsz2BDJk_6jsz3Ksf6hHm_2=yKRDbTN{+xe)1KLqUWL3(0aaLn@um=&u26+a_tjlV( zVhWfdbi5NT;qk~EE9}?At@DbNGXNd2VeCE4+N@dtG*wIk5QZSHzI)wI0kM!Ie8))^zTENQY? zm=OrK#&yuC{Y00F(Hz%Wv7B=3g3v@stw~;GB`f&4iX7B(#;IE(%nDUtNkeGCIsa6VSnT22QZBsZO>LB2&Hiz>sK5wCqZW2IR;e6fy4@Q5Jc!Sr z!ugtJ*^(wV*YaDr*58R&N=elmdqxfa-c;fSi}(1fYPq%!UKau#%%HbJH(H0njn~L- zDH8UP*Se%2%#3R&zL+(}qeBw@>dQa=2@eX9iu~zI2xHYVWIVua%@vr_EE_V=grxp@ z5XAR@q+Q;dilary>+AoCb#F?|vegXJD!g`f3^&xVWqre`E{7BFZe>>G>?OP9u@zPu zzi^h5jBChZS!Oc1A*uZmrqVl!x}@iUOFxGrY(${sv?zQ?zoK_I+(A7M+Av21@|F)) z`WYJ@y8k=oRsYB>djbE(VCr@b(U@h~C5`ww;>Bdv2oCkIf*EP{9gnSP%uo!J?xdt( zN^@J~+YZx+jf@N5qe;K8Z?UunG1$n_ z6+yS7TWMD>59E(Kx+PS}N{q&>f+@tCpx(3w{Zp%!syQ^L`%OPrhxuGJ>M(NnkAlr_oQ9zZ~`n*{)S^zh)p_&Luaz??; z3CQI3h&6J@Cw7G7fJ`f78}`7zsYNVg>A z!Rs7rCYYkumf*apHphW#DWWKCWS0%A8NWgBXCs}#HFF|W>pWOSd8FU zPFtf_9}5cB@8Z!G>#uA>Zy98B^NgwtA`Y%_dSqFZgo#_Oq{t1KL2ijTN>I9%iG*qL zFDLKM-$BIFsfCHKbEvc{WE_q&Xzn5Rk+?aSErJ`z&B=^0uS%?F%swVrWjN*ms3OKV z;1#i7oo!%j#`VN9RRAT|1BngHR+L38X`n<-!mp%j)}0@PPh-QNj+! zaz{@FV4jD!aSst@FS!j8T0e;aB5wy7Irt}W&^U6nzZZNJTm6&X(`zTMHshZ27G4`qJ%5Bv6q8d?(rH7L|@Lv%3g7u+m$ zaGDQhO^-F2I009qCbLvYFvME2p&3(UM+n})MrB|t1vJjxwT@WK`sXAEd?hmuTr~o- zw>=OkNG4^mq{$6JdGr>?@Fg*pT%?4qDHpniF1qMQsu@z@GP!<@0Y}5r`&O=PWD>Yv zf{m|Z`Q0h)u6WHv)lq|JPL6gf$?ijT5wsp%Xg+duH8GuA)S+yKGAmD{4c` zXbt#hM$pl+Syow`4KH#ih6C}hg6SO(YqW86D00ZziM!dvMgjxS}3W?h|ZSXyNe)HR7) z(dD$iLfdre*-z_Chp%Z!Q<)7hR-xp25ZG@VPr$e$6v$(8=T-WNM@k zgT>jWNSuW7x-;T4R5woxGIk?&&+JYWRo;JkVlogrWf=P;N+cbcdQVSNyKAN)%0$0j1 z))OtaQxwnT5&~LEwgs0xp6h9PNg>lW7H)T+8DG{v53g?2tubs{2w75`tCd@u9%UOA z5iVLGQ{8QdHaF@@8uku3Z6(W5lu-=uJTQp6Mi#ZHRw?WBoPR-%Xi+Sh9MrJGBb>m@ zS>}&WkDh}jdDKO=olWIb&^1fv%k55sg_619IhCFxUaFkXF1I&XyVW*wTDYX{H$9y+ zCV04J&A5{?YmMLD9&@jE90gPoz39UNT0nN)7eTDJO=66yaU#-ra1>CPsVrN8TW)JpQYmF-Gc1hsfLRWu;_TMQI7?nCs!-%5-<_2fKIT zkP$2NtB*-mr7V4|Wd2I5r0UA)j9AyvR>|CF1Jo~r!$3RZf&CHM8X5eoNAFz0oJ4$4dAVAKe6}@b@5v8BXyFD z(U8mmd=BY7Y{xz?mjm}i*e%`XcyUH+UHCW zR&(|s>{8krV?Z(q`TKm1uLEKQoY6XZW`gJ!pWv$6tzOGH7x? zq~qRe==*HpJFnim-9sL24SJS1bHD=t#2$o#xda>dZgVym&;f>lFTq*}^ckeN*T4;3 zMFoa0)+D36FNrYjB7#I$4Qf?g%flL#yjdhDunZ_%?koIpAAk#1&sQwS#%@%M?+sn= z2_NYO9P3A~dASud=ZV|U^UPKEZbgg?4#VHOJkP5VLTShaTKGUe3`%>C6&)T-_-yRf zyKsHOkZ37vZAVut(d?m^5X*j@x|$1PJeNK{QmUx zoub(=h91_3i zM+jx4m;nZ$F?msyq2b$V43T!=F}b^;Pg^hS+;>*H9rf8h{snj0K9S((NP@Ns*bWie zc-kQ%bPTnR3L!((w$3$?IWJ3VuBTzaOI|k8>g=HaP$UXbAVD$5I;W$j(;$yg8u^9i zMFzqmf@(QI3KZu>Jw6)(!9FHN3&bJ(3QpI+mx`54bYmcR3(8l2+;y~#lg%~H132X{ zx<@y)>Oa9B| zm-LsrU$&-` zQa+e=-szr({Q|k$))cd9S{p*Gs$`P-L$Og&zM8cI0c@sT=s!vJ1U zDXnhR4yA4ST~G|7nt+(h6puBoy`k3)FW_1Rja{xrHUOXZ;7xGNBZXdw8F?#9^Pg8A z!Ej=>^QC0^tt{b_d#?`;?NF;@W0R~&#VCnE1e09_bVjjyCAlUSf=FejI;xD@PO$k9 z;>pB*;Apa`t%=|~tI;=EOUFYosEG%>`ASgPTqc5RuSvZKvNpnSU zu5zg^m0WeE3CAr(M^jF~IJY0$Ffr|_UAKW8H4;9D73|})Z)M4xpyIvtb@HdV%OOAx zf&rvn2%GhC$3-gddan0z=_->6%{Ee(gUb}ZR9$Tbb^53o5`30Pn%0@WPz45NUnk&= z!608_g*Rx;-HwG(6g=uqqj3o+WkU_VX}UP<2pc6(vAfYm?1u3boC=T1yDebLj=psZ z)R`uWPsD`^uN+{%a9np%jYE zcZp?KoPCENa?ZCkAH|r9*jPfo7eHHrrDRTote1w>`6c(*&|v`vo#WH56+%|}YbC4V zJ`C_2eY-RRxLWJ8&B)rcYa`iEp@Mt5S$GLj4F?$CmCQZRrkRQLZo!s)Q!QDBVeZKW z>I8#qePOzn*&mXX-f~lON|8>c&1oGXxW1DNO z?&j)lKA+EfCwpOwfzBH1?cs$n>$$b+hNsp2YE0hT+KD{s{We~vbS&<~9*!44T|>pS z;tonjb3Rgb$u{(sO9=P{YKCO;o>g@>&LCwk*-9!#mdv!oJ{C~s&&K9m%5)J1X~LGm zO=OPhwjXS{aTM_`uo%^Dc4SFSM}FuhErRI9h`q3swV}6$i`#4s!`4}`VL4MYYj$sI z&ONsYD>Iv2l+$7;eH&Edl=sFY;t>g@Op|K9;g`qdwk zA#T=RDg=n|d(C=m=<(d_`-5?_5S_r#X>%En>>LD*<#KiDj$ipE{#JRNX5VsAeZ-=^ z_sSVEw089hTEp6?A070p)tGm6q+w1mf|Ty+l>2Izii%X`J7aJF3O#+*8#k32`-IaT z8odWK_9=KR8vWGN7-4nmMwfO^6J;7Ku3Y;KMH4J~H(R>*kdY^**k@d>j*K=FzLRJz zrGD_{$DeTQ6HI2;)>>DO9qeSEZtPX>CAc$ob6+;c4)@fj8vBTWO0y)9d2s+kJ=$KN zwI`$eaGRY*`5={*Vj1hug#M{b^X3YL88ZRUL}M$?vk><2!G^yACs50gN6o0Jg<|N%?cJ@VD3Ieg^)bid)?V{zDO6oYB8+@}JN_ z+y1}X!swEHGABT2w+G|Uy=2^T9}WjD03C|9X4O|*>C&lDpmK46=v`rIneZ_T{g(T7 z9V-O++mf#@UZT|KY?3E@Rm+a2=*LI@M9%Uh8Y=sYh5-e3R0{GKb`?+ib;Y)@-qV$k zLEwg8J;iL{8VGaPhQfG#1wG=3=AysBZswybL;~z zcBck>4tDEan$qr041tigcB5xhT2VzJ&v};N)o}^c76aQ07zH2+lyLtf$HO76Sm&EnV5ht-m>yesvDehXjUJ< zN5&_njK24eO`!Kd3ISi4ZO<@dcO4zZcSc(NGtfj1<3l6eezcBA;{82oF{4g0rvwniYH79me|kDTCH^US%x zI=_sSuy){2c}_k%u)aOv`bJDj{^GgUed!vx|Apxxf(;=%fM71j&7YvG^fMsPug}d# z&lb=AwXd1iQrY^#w}>jQXBhHmI5OCYvB78m4!3f5Y3i?;V2UTL*^QK?#%E9J^t&K|I?VsoWVe<#zuMcQ68)*DP>-|uP0r+_+7UHm^agp?g(FAr-kins{ z6Vj^~pwpFZJG}mE1b+dEK?9a=^H_5SR`Ni=PO};(5a9Qb=0(P4q?D3mQmjw=5xfah zaU|>`*Kne7NwKntE#c)rK23k^1ge2nVWM9MS$^TEIdC+A`TAD!lvElKUq~eoIb^6& zyHU$6$+MZ? z;8c_yjJuH50&iN_P!yE0W}K@WGJ3*1+PSlxm|08s+x`LDsTbs}R5{H$VHE4xbL}($ zYbF;gVOmqQ#iA+PmFIMoM(#F)Qc+GtWjY1C9$O9+@hFx`-Y`LutkUHWx;@!}mRF3X zb4*+UgPqL;tP2C=GzGHw#@`J?bC!M!=&EE2ii%K8mdq7BG!oO$=-&CVGGNei3&*p5 z?i=GAB;MRLoFqF15C!dFR?_UNV+?29tz{w$G+ou}G}1$}S|l$G<=OBjEK7sY4c0bm zINxGxnKYa9{uZB!sXwYM%rU1xsjTCtWg8zOSVQOx|Ro ziqD<`Zs9k> ziLt-K0nK4R^Jk-7M*#ZZ6A9}RzC0!y;jxx~lP%c-v+=RXmaN{7V`Y`SlKHT0&UI7y9&I^E;d@Q$#@_<4R0SW%b#6b?nSpuEFE230)M|$qi0STkUpfcV@V$f{ zwZH@yIqad8YTl&kZ@7cvU$Q#DQ!jw`m;*Ud=v4Oi*Bpb*}*EtsA)#xzzQ#RwvTX- zqcT1|Q=rq@PlLBTtgK;}r#t7k?N%^oRzWOeQDrW}-9yKe8)#{UaXcQ4GGYL>2|!{4 z+17%FShXKGL@YbsK z6N)6_xNL^qOcuDpxts(`vm=O|hUF-oF-gbpEE{-DXjRIbmOKGF8OL&exG6ar-p$!^ z9z?vIK`L+vv$4mbTWH&Bd4d(OtKFQun_3{7M342_6-XxKMlnWsZqx@VN|N%GFYRa@ zvSes);xw@zpgMlg4w@EiL7h8Uhg~BCkzCnxu4m2eqOMchu3@8Q2Llc`GpFs2WtsW! z5FZ>tZtk5IRzim|JI?xMA&*W{C5yt^&GXZJ<^{ne>K4J;YRGVr5^%L)TB#sRDXH*M zY7b-F+J(;~ePv{OF>axIJKa0>+o-W|fjjGq9mS0Ce=?X_=FU}tx9yAT+|&&ArAW#748zg9+Asf*6}a5#Hd?eNN=7tG%R zj8htXeypvR*LG%%A)Q zjSd4P_S?_&^AXl~^)bF-2xI-i{+=l)4-3^9akF22ET}N*=xfR|v(;|PYil^9aWuWX zfwRP)*$^*3O5?U=@-Yyt+)!bJ&s#1l?SeI{5-&(hSYI--;p>fiIiUqjjN*$Wg#5XT z=v$YxtaL3M!PN`i+eh{7`TabwAT?Bcy!}imHnH>D;V|ds6C3p0{j2C(jC`KKsm*7v zP}TOc-v#_o#n0Q%@4UMYclFD`_z7-qp!%`%;*!p`}kX*7n(On<3{$=}# ztcTftCNXq7zkSGUT8M4^+$-4DJ@Ixoe1__6?!G%xf3y8e)ZC^o+`X4v%xsSd$ znVR-vLyuPfbGPG1st4MB9=ASd`*|ph<9Qc-qv#WO2qSm9k4|)%;GNp*;_xUR z-%gvDfkY%y*QrV=*od`(tcTkVk8XzM2uD$At=|W<(VCUql+_3>fG#8&#lA=G{|4@2 zDVrmN3{_tYF_Inv$S==zy91>F?qVq`VM)gUW!!fW%&c7UuJ72AT#)vHC7*2KZQR@% zVI<(-Rk)PPZTC5>i&8;9GhFTN<*>t#VY3*=N6Dpy)Yl~~8HA=Ow#qO))2;xPg0$I4 zfoE)nS=O#rAEAC?5d(-{w0l97%xk_d2R6;3Y<}&Sscz?ZHygZ0&#`IQ~7zWF8paZUjiKhhNJ*uGjj9PEK(#=V88n zl(&1A#X+dFr@zQ`jh=Os)HtsC&0#lszm9}t?tLK|ZzYbzUs_srnbQskum#v~sbtQ| z4Xbnry3@<8AfuaV&U8uc=+@z&QD4#&%&Z=fxETmu$NTq5EL_(uwoJP;lVS~3Up)YW zN2_R2)7~{{Xu;nj3eX{tZq6?+Y-^sefH;Srv6Sd!!*YtEJ`X}&FvJNe4oBpPTrp*u zbkQ$~6t_&35DKyu{Ab^fhav@YbqTIiNNxquYBmB0YQ$p;>r#oJ?QuhA$ECLBr3*1j zWS&=om)kQCHq7lJl|D<^Ez8a{U(cyZHoRnsxoFO4!RIhR0an*Vo<>BDMDU5>V9od~ zchQ@)DYp8Zbz#ZxUR_<+vJvdQ4X&$=%rSdzc7=--vzT-q&4-B;p=Bz z&ci=qtAF7SvBhs0!3ussbC$iJLrBmMP74s@J2#toFKQSEDQQr|D+?p_#zFadONj^f zRV-t-h?q05I#l?!-m$T??l`AW0D>}n-|u6Yi@t!T31#O2a0~hYDhsm%Wv*`6)@g&Z z08P|m3x!9DI#{1k&ot;iKGnaJGSgW2R$7=t#Y)B5EhAJdd8sJiEaJ|B{mYkx8R-f< z*MRqETCMTNU80GV1W^eAl`<|lrgf^iwT4PFHD6aeWoH3V++S`Y7IWuULtbyto@*KO z4cqF#W$Oi)&=w@snwr}V-b-l7)>84G#|Z00C$g|x4MNPR-%o%n0%NY1Z!`vd10pC~ z|F!Ntrp;pyfI@w6Qpv_G*pXB+o-r_O8_{TI9cGo_KUJ(Q_u<3M7vv()+nP8|UUHZb znw1+_t;e!LA^WusY09b1x+2tFvS28J`xSF)Fj1@)=V)R0w&F%%j)=%^$j=L(K7j?o zc(w&0PNX(23TGw-;VH`tS%OXmG#ds9jdTg3Dj&z7nkXTVJ&2D(orx7^O8*XaG=dIC z*hK`f4x?tp6KSY`IZUNwLlsR*rnoM7f;hQOb+UAWsu^JAn(g1n!QAM>S@zq;KTyK^ zS3FYuh{dp)cJ?0wyXuqRRz*44C)z#6{lRBFSwAk@cw(mUl&sSUV~xulo|rfM zAY+XS2a8C-j7YzX4sF>ttYrd@$iDaWC%|`lENrL4;yOJhrqko#IZeQFdQ2RrN5gP> zSp233$8LHY+@?cfHa!$x)8k<^Jtj`mV_-BrCO*^SU^7L_@Q;YeG#QWS6JRlYrZ`Mb zn$MHw^Q8Gq7;7X3()2U%aM(vrxJTjh2{Dg;%}RD8w$U`Ye{f8r&jQcrQ(+nHjbrrK z7)J3zNGx{IzDae_04&qNS0)3meAb{V_Xb>B%lT%kQMg~oGPrsE04U;Wc@gr;H$`DKV8mvQ`{*o5zB5Q%+q z_dhur(EX5rrlJ5%KmdwI|CvG-_)JIsnGEDG74c_Z{Fz5V`gvHCpUG=G4c%wom3>rH zpHGD7^Y~~!dm#BtMDf`J!Dl*p&%I%2rl9u3BKAx|>)8Xtz+Y@1D+WLMnWSviis`F_Pbv_R? zozDSD=aZr6JRpM3=YgK{Ns)8n7V5DPb3O%H&Ll2Bj*_zvTYoTgoQFilnY3;bufsSr zoCwkH-bgqHOM8Sv8=3fc{E`)lCFXK@-il)u%-p+%Zy)!z^?4=_(xj|t)|)$6ZwvzJ zZ0JLAr~gzrghns`LUNE7czcZP`9AvURQ2-*j35f`aK#eFZ<+46=z%F&`4WUmxM`b{ zvL2;q>^O8QmG#Z;fp@e`4CAGX5W!Ge{D~cg5rr~V+Tgr+KmkHFRHT`a!Nw?jKa7T> z-c3i;4Q;v6NatOlVj571Tr}mH-!frN>*7c84YN_RBdx9^gm3ARij=C9eB4BTs)QecR1gm}(qx0v+VMk6&T+@mAB9I=F=u6CuI*8y+Sa9B8dcRS zw)jzWopr08@v+Pz2%lY9vhz|lQpB9R@Tc}2(DsG6RC2ADb|eF+Z4rVzh*EeyLXM_O zssI#9^Pk=AhIBNp{@P~dMMedZ-R^i_Q=YN(9%D6H^9mX(OSe=S!yv|8F0jHRO{3Y)YoWXg8t z+)YSkVqDYADk$noOB5c&# z$;+0}(6T>tUI6rvHb=9t6EjpFA;^Scn)%;($WNgwFL;}ygyQX}b*|j_$m48FDsdy^ zo$vw?=gyl&Ewm$S#(X#kb5pUgK>1xaZU zT&Mu{VT0Ta`$e z$^cE=N>K+PFe#9B4(ge$TXjzLq*4$n^}>B)rd9jiNa%c9U?h;A?(S<`qnjv(Pq7po zY!FZ8dvAd&9lx_I?Ra=qqg=9$wdxGpsmI~`)}xL*ixprb(5ggJAuQo-1fExP6mUj$ zd8y=*UGo@xi`G;dzxJ>k=HGXKfx3*seIO2=G>yWpCZG?goR-gtxnky~!-(HZ3U@X$ zLp!q>(%y^HY@)G3u|+=px)@#5HfH2TBkvkP(q_*?m^0jCs+^IWCL1p7MS`{ivx+uU6GX2ySY}aVHVsFO0RZbASsv@16*b(Pmsao8Z8rgzjF2wrpY;1R{_69K8WnM&-GI^|Bj2FXWFPWyd z!Y)5=ZpO-~Jsmp{!NTjkA{S*H2Q?7s6w&D1Wg&p8++`$K0nHU|>P`HJ=%B7z31GJNjOu7!yT>mqo$<8g2c zvG`FyXDdd**k?Mc|B7jGcQnqtyrHHyi1PU-W|YvF98E4z`mo6cb;{R*T+1KDk0k=G zWfs;Mf_!3fxt&zgvqy}j+U9Q5;IKc6x3aHTYx8-_?(89BK0uJBHOLuRRlpO>+0#`X z6RJym;m-|AZa}Pdwq)OvVM{B@v#|VBwjHrGl4-(NN!6O|D>aZdV;X7AGOUQjiI*6x zjFH7;hdX^Cks{&QrzklPA<;&Ps>O!tDMB5*-@eA3fQDc8DFcs@$ujuDz?Y@Wod0Qe zF&Yc7n{Zml^jL>2>`t`T1Q;VwU1J|8)?E8;KmaLAnry60ir+BA`7S9IfTbSozk9tl zjFgHO86*GU6X$1awPH#6XOyAq6ncn7b&2Z#;r{jKu6-SoT(E~ex{DPvUqLI4`-Rz= zV+UqkVCW0-3UW-*aDc@ZX_hr!9!Tf`u`+~@ZNiyT#mHNEWp<*<*o<648d&`au;91y z)kl_8Wv|RPdIAP8e_~hcn;QNHr`~}7scAR__8D9{qpJ3=o6TgkphLFqf+rydk%HJO z|IH@7$$ZHL4#JvwWYDWyagoBGL*0+A=NPh3?4o`*0XfJ+I`l7E7e-J34D{-!ie?M) zGJvAPkNS@Z{&)w};*Mu&LKQ|3*gD`^+Sn9eImv*l-0_>57;Htuf~t}yRYsL7m)cu) zVRVaXio7z{ZQ85i#O~E~yTSHdRxrgSg2-AMdw48Pu#AnR+}>W~$}u2+>SrO$UnBY! zJ>KbQz`_0bT#&zRU7C8o0E^l3KAyym(H4N-!Ov%B3 z3&Ga3RMv$E%Tija=FOqQiA zWG2_!t3olF8s(%^aNhS8m9Racp+SgTvn}o`I$ie+F(^p2k#`7h4ShO+rgI?+nQB6F zX@wsQPcrFO7SF+%AE3EvtEn=YzkWD!{B5d0wFUZNMK?e z{ZEI7xi1oCyty+9{qkXOFQ*`0PB5Uvqg+mR`93hRS_M#El1)ySIJRIeazO zCG(jYSh1F2gFOyGTkW!d0sT#W;}g9QUBC~uJw>-MW~F*tTe^ChsbNl{${?i$krP2@L3D|I0*W!luf2cYiAiTlUWo3&2+R&YcT zh`1ldFTOo4y_|l$x?y+s=Q+?!tF}qX){4QR;W2_^c7%GgG4<%6QnzsXFiKTFxCV_?#!jU|Bb!*k% z-A?OFE@|e})#-Gj~xQBWSDz7lkXy`@VcBXYve= z$d}FwGExRD@HG74tGzu9xj*msz6yLxfh(JKaJ4oA8rjtm?gh-mGfK~vZ>$JhbGCY> z{g#DGnR+8Uv2p-W)0nQ=fVQT-eb0qAq-}e zHMp`={Jeo!_ zjzK*}S>iZyIk5$p~w#Kuy#mMl>3+{-|IaczuJKO$Ss3^e z{OuL;aA4dVEs0I*rTG?ChZL}(L)uzsOqgfvmSx^D;0iAU)&SG8A(#s@8_lIaiW#ym ztTMW9Bm|QCHN+eoj@ZNY21(s??qa6eKoO+>5M+)+`PI|8qnK~*A3S;zjJ92G*55o(1c97+&F z$%QUW?-*P;KGE|`%bDRfuHhZa7j|L)js7Nk%Q04`=XghuT=n*~D>J~-wRvm*VA)~~ zj_np{9W@IjY@{v)&DqfmSJhA$o;f=SX^SvNtd2-$MpSEe=zrPl*$8T`*Pkuu)*iif z+cEY%(mTeQ`yT9FeW-bPLGTyN9M-bAh_IefDjE7g(3UfxC9~aNzoC1ep@)$Z=PDp2 z%QBm-=+#;y$(OR$YljRFwaiDmh1BIgwF3{nMJ$|8Vwf$eg~9wtWfz^LFr2d-%nw z>+JfuHcnBDseH(O3$p-=_8(3IOZp$qzB+q$V%FlTQ?n|8E&Qv%7Vdv(M>>FD6$h6&Dn1L>>8qRZfNzdQ_`qSL6Hi9(_Z1}t+U*uA;FM?(shnWur49%Tg zuJgM(3lA3Y+D>q{+=4 zRVgH(6}03_p7GMihCS^EB`xcy$qOs1TNDMOBy9@|_eot9PU|hOFsQxF=Tes>VajO1 z*_PZX+3!cV{_f>>&&jOZ<8~c@f)3j`@WX|r@L(#f>m9d=rYu}z2K$H7ZQdSdbIyuGEb$oSmB}XlM?#!~z z`GDaIRX<_ghYwah($J^UNK7OnJ{e_Br94YF?o$tihAhsc&$16GKcHbHSB`(4nU-Y* zTRtsl5Po9qi`DEq(o&nw_c@gJ^S%2*;-YClYy9`UI|u@!pF6CEso6VpRE8Slk*#d? zohU47_8=XoWL8Z|SND(N9EqLa=vx`=!Oi3T+dhQL68f*bmR6j-^JGX>%eVd zqh#H&8|DQTrus{)s!9(++bpdbxy!SE7EkTL}E~hWL9L zt_afOm;aCVXPf4ZWrC5t-wu8b>x~Uwf4`;V<1cBgw`Q4>5dt7Ym2yoq21Nm5!W~<13z45&S-M75>q-VAgW{Xz`r@i}EmEl!JIim|;{+!^& zZHc_xRGj1J7C(wy))szm$R@?h|!Qd0b|ph!2FvcL@m*1wB$ zYWHl*>WIwyx%{Ba>W%tC2rOq*! z-ywX9>UhYO1(LlxJrqMzHxXsF4y=zy8FlEY(_l6Nb)OJDZ7bZgZ7S2;RNLIuuZUeR zw`FjwthR5jXmX3mSB`e|^#yBbj$}1pqkfLK9Q;V2`;uteT$7=cOkeTXR%}45<@OGO zU-!fq1*mzI>6@yg@w{78B;&KZdUwrrZP-)?5})$xWK`{!(rd?K{$w%w`9gUeP!kea z&%N^i`r?bsiQp>;_*`!f_E_e}tl_J2G=n|}dKMAPy8un0jo-Y!!vS8^PtP!!-zl{A zHq)Y%H69}<9&onzv6ag0;M$sr+);`8{SIM@aJ8QI+1+TJO4VX0mhOw>s)MbX9wrTh zfkT6*m|ktq&Q5>%fc`w0iAC`=wm+m3seD=q(cNPngVLn@QJBiE9vDCU5~+xh{^f0> zMQ~k71|Pd)+(}Z^uY-2Mg9xTp%0>OjtARN%&9ZFLYbc$d} z{#|q*>9ga~T1qJa3A~6)a1W~`iFUvMYiNGlG+U#iSP1%HZVTGk1m&!Ok3WcSKSNzS~LkhawO}k4>aGgkk_) zo*)IhmMeB01R*0<7%$J}zEY))9@vxK-Cmrm= zo(vuq;=CiW({E?`uU$IJx3N%+RHIdbYqEszZU_(5V%}K%w7GM6-UeT9>2F-K+(*y$ zn!K~J+PLkV-wvWGv~x9vEN&k(v{#38&ydBe&+AfGB4UrT$Ky}eeMR{Hfu!g~6is6Big!DD@lHnCyb#_99{Ixohe=msMtjMJ8)l4)SDf#9Mec*%)0%L8m4+o! ziugLxy5u#qL*F!*#!YivZZGdrY_Xwlw8Pn>zn`hFw-Nxt6m}xpM$R;~czK`cM@m9h zY{VpW#mvQ3nZ+X$5Ni9?3o>-|@3SVZqx9-Hm)xyfR~x;?RkbB8MBe=E=UuiF+H}Za>YGClQ$xp_@WO; zXNnJ>`dB+NfJ8H%hD)dTTN|y5U`Zq`X;a%-t z%5=l4XW|OJGcxN1lQ9h+a?{hFxb*hh(j@uXFdv{;9kkvhhv{L#(f$V^C7FTVx)e`3 zMbz?r%`)O0et97$O-ALt2RhZ?yBg?qfn7`)om4;vuGYmL+!x6(-y}o-Xy&Wn&|MC; zkTF{}k?Oct(YUV;y%J<1(O}Shdka)^pu2rrvC==LgEC_^q2JL~C>pQ>$cA9AD5TfF zGclN1gm?s#r@0x~^JqOackMD$HGqR^Rw+SbY@pC4WBJ(<#aC1jGE&ftXH^~JoSq#1<;)vVjGMx)xxO_-4#vd2_EtxBnh>fTJ0vaUtJtSQPW z2eW!35dgR@nlqdX@*99&Mh&o+B=V5QeEeWwHwtT(a8ly}aFw2V9%_e4An@e>B1J&< zfS}&tGH6ZuNK%b2bx1G!)WMnX>nmi>EfZJ-Ex|0cvacX(_TsE-exnhWw0Oe1cvPim z+;qmRM-rF)X@>brs-dvmg+d5?e)ri@kTQXJ}b7Gn=js{=irfwUekwO z?fPhMZ!YX0_ONhB0r&Z_-u(|BvRm*0x_A5mx_j0E3zlKP>=XEb5vl_ytuewt*=ok6_uw-?6bDxX^2kajm;Y|AA}tBa^(xZ#_K4<7WucfUDt zm_<}eV^Bbpu~vATkTyaY%hfNB1hCP_6USDP>G5&+Je=DDTLzKwr6`IkK=bbgqamkK z3;$HjG|xTeMLNC2`LAn9(ktWfl=WzAb!;-#FCdky+|LA*5-P2Xk-eL43MiZxSNb)q zmUCMV42~r)jr!nVbpX<|#gNQNCiq^Vm-Z1>`bl66mrP#jk4L=Np8VzzcSXZ9>_) zL~;VJT=D|ZM8uIeQM$GSs;+7v?rK(*us3|y#z?|(PI?Dl-F^hSk_8f2UB^_+(G2<0 zDjWB{7E*i|L?W><7VO2as=gc?pZ(>%whu~?Rl0m1w&>wwh}n9?!*Ytz)^@-~Gz~BY zVi>We12YgrIK>Go;sHQw=t5tNG(jP?ujXxLLiidCG7T+&9|+9*s*(R$`Qv}-&*py_ z9D25tp9~N{u_-Xp4kQ(`z?EccFAKT)HGEo9qs>2dW>sH3DKUc6vAoE;`5Yj~Y__rE z@7(nx+23WIQd9cG?8a&o4bQ+HG!{WbXq_}s3|8;hSK^xRE0b=N-8`Y4G)XD^sAir} z$@33_IDg;O^o70I)9P(_W%Ory`s_IxHMY^tS-_$9s`Ssh*HG8gLkS@5Z+@G1bG^9# zrTonEpHr{b$1Imx{ruOht#sT~a$p%!dQ>ntfnIh++;kc%w=iT`jy%zswlP?2^J$}- z7z9%R(hc_Caqtd_f*Mr1R^ypbS1&rC><~;KF+xECsUE10KzX3jw#KxFnRU#JjUL2U z1*EJ(R`uIm_dV0;jDA6IJ@?z+I;`fK(6{uq&r6r~pb*Yb|6aD}Rq)`AX9wAsBQ<7` zOk-+xxVdW7bzc{5sEw0c*ct1Cax<8|_IChquW{clxRI_mNfiP^@q4KN$D!5Hl*pjd zQH~xgXo=bjn=}>8=S5le&a>Pg*f$u#Y=>~kYkLv{e|9^a=?yO z@NZ`54h*(ay@?z}8(Lp9MSTGTQ(A?Um>U5D)+o-VDh;Y?N9Mms#>BA!uIp!KL}Qgs zh=Fo8Lqjkhdw zuCsueRG>8@@4?XDQwaZ7i6|?xER7@DG%N1M>2wDC>kW1kZN7>Wqr$1yKmiv6&oOx~ zgt9WfO!G=N+n{Ie^3DrIt7nH_rr#b6jqh~mZAUpiLTVTI!pZPVpi(r&4f_Eq;T!dK3H8(=(bsTq*%x%yG>;lkbV=Ul-r8+l zmXV>-*qsnnNhet|Y<*c>ms4(ltd<56miE9mf&E0_YG-9b-66XRQzX=jUPqPV6KW+D0OrA1$-q@N#R^HZP|81l%x~N8cQ)K&9Oc zP`S_FTPM|)E)0x)o4wK|r98w@Ir2V~ND-OrQ^SG=qC#M2!{ z=}S+ocZ|sU`0@cJE$2Qgy~Umx{l4`j?+nXyH`3TB!j+KJ4anoqTY=o#9ByHMTayN? zQ(#5MvBUisxJ$?-5VEoVJ4FzlscJK!i7_194bbAt7b%FOF(yB1Vbb@41+R4SIApQOX8dl z!tx?5_~k7GpLyrRNK~K#2sGN>8wf^yLH@(p0*kdT#;y!G8Jo211-@?ork11_J0AIs zbT;EwgI6cni3vE3tZOPfqFdF`5=VVv2Z&11`<^2_Y4waz&W=Z##dJG8#qH`8Bn6sH z@1IpLHYAoKy71H)NSe>v+%4JNF)Md@!vydy9$nSkz)}a zNW0Zv`%>X~n-4TtjcbF~M_-JH&M_xcU{9l!4OaRvJ|Xt}lkT+S`2I5-a|fy2-si(| z(@S-Xht8{0yHRhhT?(HqWFd^a`F%%Z=5(^jKl_igJoecdzgWA%+E6Cj{a5;Z6_gViaoB|iXcsXHbjC< zC@4WgS-F0MC=?nNB`H`Ns|4rFQyr9-MC<E2AzQ=_dY7FtZgpfVCd#C~;Jj$LqP9Sr=cI#GB_GQveJ!A(nm^=%I*?Pfe7&xWvSUQioL-3nP5jAU zju&o9kW*VF5?1SQY-$W++9+9#hLyt>z??#piX%jHSGy_3M7E)sOBhX3Mi|K?L`>@wN3weAS10M-`^u5KsghlU zPUavU219kAn4hCxXCi9v)Pi$U~8%jKyqvnW*$aI<)mNCy19pv&pery}RovSwM z+Ywond)KR{A;#VZDzm+3I1V8msy#)Ab(;;l+u!#uwW6DYX|CD2dw5T;%DaIM<1<`5;S_OG-ifx;9&z=-04FmAv>fCWP*8f}9S6W>tWHvC25!13C0W4)y2_F2 z-#YTAYxxf2&X8wX-RN7}6&S#S1d6aUv-WX@rRPk|io*jezBIH}?f*-yw$K9B8NM`i zK0W-|{o2b9U22R_B5SV*r6FU@GVdK~2!(O#W)w%Bxak%kjr@NP(%oEtYd{D(qdy%! z&9Z*&n}FTE=rcQmSogQJmr%5C84?l9hqI>ogO_Yo$Il(tWp$l=J%EX0?cQ2*Y-rP5 zrQ+8vcnYd^h?34$Gg0SF;IDrj5vMm4Chae-Nok;+rZvUwoYaccNs#I2f6L^W%E&c% z<24~nmUCK9nsZIQ5n~3B9)Oz--3=>tNpq}?SpYvpos%iBeLOga>$SFHkcAV$QoyYR zQVd^m(hLS{47D;*dzrDTF`Vxd#SjbJFf=7S4Tm!3w?K5ra3tq^rqjnFai%*4pmFe8 z-J?*Cd2fPEf8h~V*niH+pfYOyv2sYnqeoiB#bIq zhP82^+J_>A^q22NLfCcC?O2PARuX_?^o>g@N$|<4Q>u2$04N-9ga5pso+T zXG6d@lTPG#p~|u72dj?WvltaI{I9|CCeVk>ZQ8;-8Y(J402 zy{cqS>z8Fnu%?K4-58Kaa4NzfahGGC2L+X(y-r!#PY;cBD_Y-n0zx_*Q4YEFP`? z1F*E79~Zw*pziT6fR-@dj_*8X)+;iMqSANNS}1z$tsCHx4bA&3iqLrL7A}u&_SN2_ ze)ybx^bYac>(N^P0WfazUF$8l*=1iom6R}RK^=wh*sC`l8aBb%g5I-a`K-$YoJDx@ z7QahBr@ejN^CbDsqhxwqo<23*q$Qo>g4vTzg$v7vrkv*tK6{~^iG|KfL7bAF7gji# z+c-Pypj0gOiHwNd4!=G%v9Sg%QHRnkLAL}$L57mQSru`F0kiQf&+1vXzYl=`DQw!Y zR76fI*WT#VG>CWEEHX8kBC6 zrX{*DKB@<)mC{xBwr?A0QttP)D0T%({#r#|tK@Vv+Ym1j{ovnYgpz%A$ixwb%mTDk zc}Th|Z>~Nhl0RyQNFEERHT&ae zl_UB}DROh5==U_`Aa$EMZvFJi+OuUlOA2i_{+{OX6BL&P1Ae-jmKOM=UI>Y}q?A=T zz4JDI+9{Y%O~-YztNHO*1}DWjJ9)1R!bc<2U*@r~YC})Y3hVgYLTf4+tGB zPT54K*7Lq2&?9&Qnit(jW!rf)qAWHZCUK%#aKLc<&>%amav$3m z4&z@!+F6{C%|ypK$dhVu58*!S1JvIn`0WiS*loC*3YuGYI($6`85!|X zXb&QO!zlPgT(VSK2e0|){g<@^MTEM`#5y8>_1seZs$2w++; z@Ds;AD2K|jVPs?o2!#HI>E*t_k}ZqOMy{CMVh7CODd?ME@ZunwhS|R^s+wFAg_gSjCBxfw#SRA4``U(x(HD#EGMD{!pM#0tQ!c6S8?{@xL2zJLsEZq6 ztGSs96z;2PyO{Kf>ijO!L5E#_Gr2W72dP=D3*0+lXQ(C>Duph!b}MHk6tRfgY{X3# z4PhwCub=RE(5^mWRbp}3f)x}0?)rnmiiLd?K9K+&HDG8Ebi*`a0%1p0B$Z-#BuABH z91>#~LLUWoPA-8hIPW!^+7H;h@!OI-fNyq7~G_umY9&Z~SQt~te`2c#$aWf5Thg8UA3Y>A% z33WECOX|DRX-4Rs(uQz{Rz>0TMwi#Ymdw>`BP7~v(G9}Qd86%{AG;F|eujU9ucI0{ zxUX0Kh9Cj$L$Ou@7%Pb{ukB^K;M&PF51qALW2|fK9m1*1(1FopRn_=-9Vz>`$jz1B zx`XEaqiJ|82!)k&qe zSFHNfVtO{^D75hP4)f}Z+-6!pz&i16>EFA8m<(_u&`t%uB9&hF~)IBjx3sBjBHEs2MF`}0p{;}OX#JsW)xK5(5V zJ&QYm#i%mS{)j-5yu%oU(>O#R;$}rI9l;EyLL_nup4|DSD27fohA=pBF+(+&+&GhR zR+D@RX+@PyuaB5q)mkmnEu#E^Tv^y*Hk|J7lYuD`zVojNMTwmbeL*-V2^ktriUtFm zf~v}hlD4UpzP42(^04Fd_J{CxcK)wBwqBiO$M3A(n%-n;>01vj9^r^8vOlE-i@%d1 z>NDl2E8d9(RztS7?*V?8;xLGK*0nXtqSG5_P3<9NGAUtsl(KSr*h6cObsQu>!A%8M zQhg`Ip@Zd5D|$3%y&OT^CbYn0VrN|-exawHDLJN$npeBXbDh!Wi2Y}b@82SKeT|jQ zl2QYZNFG*zS+j|PIiS>BPwGy?V`KvIM{umET4O+Vh3(NC0og1@$OX%GotAPvX4$LQo-lEFO|pJ}OZ+54Ro!!pZ`)2o%N?}~B#d|cVi)CG=)cR76 zvz7Z^j^9Un#;nz;2|Xf&JccR=RG@=3>-$EV0~&-VNg^S|#9jYBuh7qvcjly`e-W@0+)}fsq#bXT}=R{BV+! zU}u||F}gL-(B5ul!_U@*(}H1))CuAy!vEvpi(3ndxywNg`qS6>%Yy{33_4f*M_W(r zqX46ipQE+Lu*s%JqA3D1H&D9vCfsVs8d2?b`xRUL?~j&hcoo!r&9pm93cK+c!rzVz z(Q1e55uI&*!AWmja90m>^0D;PvGHI!D8l^horf#T4k=Z7%|J$#+Yy*j?s?Y&lWs3p z6q@hAxM1CpKgVceGZwqGvE{LDmD~naq^??nRvk6#f9K-u04VsiZVcC%Z`w6MUPqKQpYx-8 z*o%~>eE@4BA}cv|@-`(|)9Tl7$hm)aobXnOG$^2?&DUh66$e`01u2m{Dpk;!k;(Zc!hYM4qz= zd0UO5L=J;FnxFIA>FLWVZ4S*R4`7CQ2mjb6kpCc}13TjeKCqX_s>S(^idI1DRgO99?9LyoD+fbzvqrc686#dr?!jqIj23x1-L6cOG4O?(3nND zZ@B%gelP!fS>nav5$Aj4`n?cVB9Ld{$dI1LfM%*=&9Z$L2hx0(i^aaQ(C{!&T76?* z#?UG{x(uff>?#|V1N(33;kK2$a1&M|hSZW(;@#8FQZV{9tTt~l>LaxK8F`T;zgg3w zG5VOr6B6B(!7&^AE3~-B+#Ko@xt0+|?%yQHH2_qCUoFXS_=2xRvCxyU6K_Yl4!`N2 zMK(R;V6lc?u03 zmd9j&5WKyBgJe~6JL||CkngIXfTj^hH#S>_^SH&vqGc!ZC zXkmn)f#aB9#d}B1#TCq(=(VN%O?rpsM-IKX=Wwu}e2IC+Zeha@m!2YNw^FvROrU zBcFplThdxmQbXHsZO`AIcy*Z)PG_Y+()o=f!|3dVN%v&`Zq9!<7l_r(h*m6V>sC=+ zF&>060IgR}f*X5z>)e-=Zg1Z6Gn3u4QPQ5xR_6Ev63TsOm1;(_0g%{6NFjIoD#6_1 zrZrl4`sHLt^joiv%@V8i5&;2gp<3EyWX3i$%Y4AsxC2FLN>N1KM8>ZpGTe2l7gOQ& zQZzqd*ozS{+kHND=lUa)Iuix^*9se|{s;)XxddL?-5G*8{L1>3Xl5bM=HGwu@YYI} zm)JGx+8Pki{SLLzDzh~Vf(^i3Az_*s{9q9qw9Uk$$M?|17)t?6b|23Fg2UC z`^uf6;eEc|#wB?UoFv^C1m#Je{dlHRJD7mTAkA8r%@ z(+JCSGBb4%Eiu_<$k0~xcJZS=$Y~5AtG@@oee;&+j-EZK5Bar)?hY%0=*+^(=+G5qk7I)P?U6gb z)|s(GCHcln>VDx(^-vuUoG62Pze~buAU;fG8OXX;lf@5$*>geM;=UqyqUj)(W)h^n zQ>`H!4BGoBmB-$zx&;+Yt~7_88Q}uewQ>NBc5K0i^?BE-x^i#`Cr%N=$a$!CBITb} z=7^?n+~a-x72TpYcWODkX5bb{QtFWAMZ4}T2r(dmVheFp)&;h&s_x|TlJuhm*|-dRJy9P@XJvU#h1RQ;VvgBySn|rC|ptv zY8k+K#&dYooFyw+S)$2Us5WIzp>Czw23y}#11VwvPZk}Ob(QsA+QhMDA-!MRWX58O zHopkfwlQYdScrZ~w(AgeFEX=ln06-i4&L$*RCjv@7nRuz2FDrt#&g#oGE(3JxgT1t z1E`{fq0)5&N`NaBBEUUI_!P;}Gi{l9a}MPJXKt|os?p96fd2wIxbg9=7zcA{5x3Rq z)(R_lo)Z?-YDfk{<~G!GkD4tL9~)EToJm%N`e|WN#rf5r_nT$) zO$b`_o~W2fgk0tcqU0=JeF8e#z_0f>gOc@Wz~(M$dlS031>e!J8+2nr<-Ny_Eb4IM z`S6k4B>vaYPlfDsbkym_5ueMS5n-N_78*3q(5jqU6W@k&F?EQy{$j7Q6=x0iEQS0V zeJ*hV&V6mhwL5?Cx5AYpF5+SuCpUS6M4qL!*SJVb=+?uOsPAWfjmMBwUUR=3CUWYO zi*Q=XAD2uH0)m5>xq7Jg2YN3{W%J)ugsm)DXNhq6ND6BuTZRn)ytm&jng+%ej+7vW%yxKje#jn9F5T!3&15FNe|FRs?n2!^5g+|e^ZZdX6zv*rw^mVs&8%MhE6lY^!U zBWvqkw}MkBJ(Kp9EAxtneN`Y|a}{GP(k~-@(6re3q!#-=THI{E+!dTZwjo^Z)WH?x zV2kFrB8*5~*zg7CB}3B8dMOJcxn5y;JN(|aJ?S8Sam0s38!55LjZTDHq6*=!Ww8{m zU!KUQFtW^5S+h#Ustp)DW?Vqz$+1lrQkojf^dbh0Ew=E1v@>-R)gunrnFYc->s9XT{_6MR{h9nVj=J~%gC zxVD(tHk)X9Y&P$T&L~YA>4FKV-sQN8gW`wTDi0Wkg#6OL7*qfqVI&GxhQl`?g@ul&g%(Ohxnw(;0L+*Vdd@tM&k$ zB64Y}7D?dR;hmJa`^2S5ij7qDav+M`Et9iZmCXmU$Foumc9i&SNIko5F_zZpYvaUNC zKEeZm}_+Q^E^G$j+P_aM_PH?>e?z_Y3OZkj*$GvGCJx!o!>@96jX82Z_*3IGK&m4&>ZFo6FXO2w`ff@~? zcB{JyYjRrVG^eDgRF5Ax+miLES7SCOL2tb9_hcjFza-(=;~3rf$hqvAZwo0&i~yBD z8=>PyT8QWkT`#g>(cxGUt}k`9!uXb8?4w7&8m`iTdcG_$vJiLDi>M@&9-=QAMQL=h zNIcBBgKP`j3%g-jrg{xdK({gYW%^!WYtwB2|69GRSMYIE3i&1Wl(jUcwXXhlLQIa&t6@m^^4{5|GV_R=9r6mno{OPa|+S^fEy&^W<3RZul3Ns zc?Qa2;Bjg2lsgif9lR(fZ9VZJ5>#}FU$Q(Xx#AE3?awXdhvQpuoUe(DE8AA%Xc3<) z)*bU)*0P70IYy=kILwe%zi}9`AAW(K#DO}G26PmuI&3;O!wotkq;o?}*?PaSD2e{J z824|P<1>c48w#)Ku!purWPD}|ouqJvpmN8PB$j=sZ8Zfjre_wI)<6otRpILiA3D17nOvvB0LB0Np} z(gJRIE?iCuPbTRONb$4`gfzOd08-)u%lRCJBQm3TA|p-yxd^rAtlM_vIT)jmAiGjH6>k0(Pp#?P{8YBB1g}fq(~9&NRS&!~QD$4MLs#S;JrL z6Zsr!x2QSV6w*1dj@JfzecjWdkj?pi*-qIp)Lm*?zfM6J89j_S#WNV z0qsdlkhusaaSmZW^`CH(!Fqtmudf%#@${Ix4@i7%OqFARFS*zz(=TIhY<*wZ6OuG5 z9jG%e*uKykJv{7t=}fX1YFWKrV))z>uI52uN$~HJOQg0xI{N&{#ZF#k0AN%`gLm^({ zW;&J~L!tvOm1UbN^=L`746;N*w zFgxbBsY|y4WLWSUN*k~}S{Nj~2{HI1G?q$J%@_)RrwA+HLi2pq7A!HY7U{un(R6Mx zGoSH=kM(A-7k5CXxnt=WY=K-NQ?6U$T<;o6c-pGyL`q1o^T z;vL@TmS%kkAA8_S(*;~)#K;R%CM|PJ9r4X?P&neIWON@FP(KUn*n&My_AQR4HMH{i zN4Iu+<;$JC_;xk4{C>Pyb#Jt(>t{9Qk2z+qQ1VEWjS=M-)qe?WMWFDR3P+ZRCjX4_5G% zyLF*MoPn1A`_$NVJNhTVyq#pZg!r|~1|t@iq@Hu)HKR05+FZODu*d96vFcv#I!s5% z*I=UbkWPgrl0T`2oy+sk8saq}1V%O}Oxn5O`57wwiyq~`;5UkIFwqP8%bf=(vOLdH z0eZB3OUJA8?d_J6vf=%m^dBtwLrD9I`a6ttpRRQXjO-eJS0@Yos>mpd!P;GDO}VQs zRrK+{KN>>m_Ku5O5HDN{AG==_q!kfupXhm+6{J(yjs_mC_MI?23@{UXDn>bq(}Wi7 z{Z_l~((o*0jAB zBW=Pq!tO#qxcbwi;lFsHx%8AjnL>Bx0SC8=G?cj4x?1W5upCNoQ#0Z0EzQ}~NDa^> zZ|v~VBdUoVp;p6Go}^}5f=rpG>`hs8Qq{nKDk=T;)m4zoNY0i*d79IOxaPS;Xa#5Fl&)<*s^H3BS`*T zDmcr%phwvsA?h)*In2Vgg`ZhT*18gp+FBn}8U~klx^s$6!hPpPm2|#`#RQ{&Dcs2S z>{X~x9XG-b4wHd!icx=w!4UHZVCMJHh2MRGAx_=adgp^+^^(>+E>_I$Ozw1AmUqvc z8l$l;onOk<#Bkmy!P21YUh!9pM7@g2Gd~yi+_Gu>)$?;lc8pQ0cN(5=^^7)u#1?%v zmv2VU&ysTd2Q-SIw&@;lQW+}&L}C#eRuADg!W zp?kK^&RRPJeE%_8Zgfda+O-C&D7Dg1??l`8{&a@AI>N36{Q_Bk>dt! zhX9{)sR}pl>)7y~oJN`+MvC^ge}-_6Y6{?@Uxh9|Y<#_-`>b7W+?As9uAOy?sHF9- z8b(9;oNKs|GT=?UbLq?9pz5kjU-9h&;*Od}_tgs#ZKvFLkb+0q4ubXElxO9Wn+{}r!yyPP<=erVTDzh7=x4c0Un_x8K^ zWaE{e8+Xg=rVZ)mY@ROF*2+Cqdfyh8lU#+b`}}@;?^fZ30%<-B3ogmz25qA;)L7nm6;P9 zt}LG}V7J{}Dn5*g96%Gx*St1cYoxSjU7Z?O8P8yD+*5+H{0=;}G&sZ0n`TWe?=;?1 zdYsN+w3Ua=W~PIYWg1`q>YL09pyDcJH+AkjBC#J&&LXDy8T5lHTmhE}X!emh+6hb) z{wUUGYUyQ&o$jOIgNT-<%{F!a%jZU;ptWP0_YAySz+aTH25zmmvns7T+Tw-0%{2W7jQEI1;V z;J!FN_6%rjhB)0YHG2C>r!GZ&O{W0#*!yuSJTftee5fs`-~!LYI^&nq1=LA*)AWJC z)zH7$VT+;?A$!=-0%(_2##3Xj2EjDMFhU#EbUF$O((1Nd#Ku&jpdq0(6GZ0Hwj-zvh$bs{;=_~(B*yyus_*c^4%Qp_%*waMEIS7-=gKC_+b6xb8=sF z0W|VnUl1ob-#It6mp`V$r9O0ZeP2(U;Ig`nE_5^HuGUXt!5>6)qocDl>rlbyucNF0!$ZgbEU}A2H zgY~x^C;T0_E9jIkyJcI2MMV3td%v!|p)u+1>+u#FeZ|Cm34)X@-O%%^rPOI;gJk6e z7spduM1+L#N)DW42*k59;NB%3@e+Q}Spp;%Jm~fbdS@N`Y{nSuhR0Ffv}_U!sj&&v zv>UHmO@TdR5>{iJFMw;`fU65m0)f7G7dz$;#8G?a7f!^&lxQweUU}H3XD#Tk+~}De zEC=mg0O%C@>H@DdEmP2J;^QyQ*~CYmm27jw5;_B_e~V2y0Vkxn(;3-EI_!U7!xuPC z3#e2at^z5~z;M#$H-H{Yn1T-IkBgJ{>IojuCdE{VH{Sb)5|y#a=Ru}3X(@jU02=sa`1O3~KdwQx!@$9ISE3G zi@fx_<1XG6dF%STk9fpqoR4_$bBnxwo|c^zT4$Vj?U8`UWlrdGJrZF9q2!^pm6fp# ze$iM*gUFyoS)R@+L)PW(piFd`4-nK|m~;^@9yuV8pM{OobeEeRn1wp@>#~Xz9)ITH zB#m~W>mBinwPucgqeO|<($o95lQ?INSjpY4SG;pR3S0gUJ8=KbbOHbX+*!1QUdS3r zPgG3g2`Lqo$x_l{k@9l4MO%{&S-H&+*NIK%ZOip^tF6h_x{aQ6*6#nDkId3E6()ZR9VII<_bMqvNIEM zQ|olG9)1fD_AH76G_J#3kPmse?*eZKUjC7k^#XaM*G-+DXWvs9pBKRCn$ZU16NQ2EX5ZJz7xU4VA@- zpkVTxn;<{#Xmu_gd2Hv-w4lB5O!acqc)ju7gB2I}X|4w#+g`6#!7WVEY4@wRPNYI` zs~~pH({wRmn{)@h*ci0Q-(c`pZw51{o+EGFVIiaM)1Q7+Wj1c6?sB?mn}pP@OvHYQ z`n+y+6>I0@{9pkID=B|^^vRU&QzG@##gUVEgdPFMlq}UjI#Pa4eP!y?`fMY@LC!{( zxz5k}l(4nu`Q|g2Iyj<$+T9~!II^1y>31G?cZN4>=Z5Uza?7Ed9dhIST48e;$=lWpPPm2itu`AnMlBBb)sn(v)dfi4URj%TnC3;NB%04QnOVKIc~;L=V)mn#~aj@Hf{K^*bMVkW}mend*KY9+X)T6 zt*r3YHMIRI7lSFVT)Jlp;i%b?PPk1O^H4}BP?7Iyg~WA+BvHw_-sewrm-R2NW z5^DpVWqt@t(Oe-=W4ewlpwAbP-ofJFJnJDC|tv^b1bC1n)Kyo6iPCIDc zVQyr3R8em|NM&qo0PMYMcN;ghAbP%zeg%f?dt|3o#fK!vW}lqfie)D{?T3zJr_W5U zi~?4HB%-Q91wc_^qTDrq<9@wAZGOpJ*mx9BRir4t;-uxZ5}O5JyN5 zI2;}P3mSf){x|-qgv9(`hVR{1b8^3vKe#BE#FDZ?Om+~$T+$g%dkPyeg<%4HhKJqeKA_0o=ExkeuKgopSagN#yphSxGpSj1MlWDT(TW_JnNB z3F=8+k=_nM36?lx^Ekm$W<*uOm1Geugv41kLG5c4B|QBq$1+)THcyD^lakc6?W~rP zGGi+du_9t(g05(hvMbSDm5MUMt4OcYOE-|t%CH7Y!Ubi;1nvI&S7;QEAIHOJi2eQ)%|)u$=J3mS z81L`kl744DEyp{!EbE`c_{+n1w3CvO6e%eZO2lL*LJ2EUDxu1p;~6e+ObR^BNV$zTu<^i3bwS^>xf<|!#i`nPIIG9p``8|Alj(X8Fs)hm{g zXV)d;lJK?t`IWZpG-wj^e7P^q314<}=SptrqplYMuBPN%Vo6>S$!Q|m@D(Z$sz#LS zDz|miZqVxSD9xX;LUNX6gulcEo_Eb`O~ozx?`+mJif1xO*LAGQw(AROc7Z?T?0U5h zJ``@(fv(!s!+EkGX_c*CK38G8p6cpBZ{9$lta&W*oq-|x2V8+U!}rf<3D z-CMqK$M3M=Yxdg^nrm@Qqx_bBnzd6R5>Cq+YkSOvdMA@hl&BMtNP!g9!3!dZKshex zOi_S)=o?M;pcIS6lwqEVJ;jw2sp4`BB*d$X2$ZmVN(+*rD=HT#Wl5EjLNjpCKo3PA z$&ewT5y4`GB-=qqGd2W~B8?=ANRj#jGW4ZKIONs1-EJjfEgm3^F!nV$i0&#IztB0FoTjY@!yGldHe}RN*vXyi}Y=TwS`Oql@R4O5$F6;wzXm(sQT=A~k83-WVxu8Nwy{`JR8&PaJl z;*_{H_FG+9BuvV`@<-EEFf91Y>3q7XtLBt5k>O>$!0FEmN|qhd`7cT5EAB19PZcR7 zE#fH^f)yPt{!>@@8p?OpGGqee4X}OhB51g+)!LJ*88lp$8BO3W?l`V$G<*~fXkH8s#5+5Ebf!|h3g-YvM6Od8%(bEMk<^)Mt!ud%v@K-e@%V;@{pA1TzT;e(DdeYsevJ#7uvum~3T_km|+%udBwcbV)aSFp94abLH zqCQH^*_I3ml2n|)^bi(mF)tqrqy?Yj9fUl|865w_^%~`l_)juQ3Tbv!Pu&Qho7=Dc z&_Z{5eo4#k2&c1EP+SRePWX~0W=6x&XvZ{r!4jM;n2=sI z#qFLGMP)K>2L&83XBmkv9hnEhXFazv!v*n#+z3gQ5ngKp5Q-3)%}63AsP~FJyCz8` zO;virvRm4>8|m7Qm6@5kZZ+l|n> zPYc-kf3rKU{sbG_+T37+?SZp@x^}r)XY*F;+B-h~vfUAX(s6Eu$1VM~$8~&vT*v!c z9oNye$94bnIhrU|>HPc_TfbxEFSpNly#Hm_gozgMteNk@ zcGol^Tpm+CK?5R_L48vOLS$Z*=ZcN4yN$kxOHjTB zphb24y7V>x0g>|!T@qo%SqOA~LfsS~1UbQ)R21q|H^={l2j1^UhL&Q3!zx7`bI5S~J+0Yrt71g<%Whj+Sp zo2S2a0d84>we9Y*b61RCl3I_isMgDU@tYE~PEEJn1`Zo&WM4$WY6IT|Z{CVtwtVJq zAs6@*C$<-yYB?;<&Z>-fho)e~8POt}w`VWJ1pT8oFbgn9aJr;|@!t3DJS+*HLZvy8 z?sJ52Nx$K&Dxt`$&WV)UKCXdt7U2VMAHL z_zOHGjqTwiQy*zD_f}l5h^_sPiwgtmqQr7>aZ#^}#C%R9RCFCer41ud#S_%p!Owl1 zC1+FeTdn4fwG2>DFZaa+A&umSRJiMuWi(lJUsr{({#H%N(ecaQ7NMkr_tx^<&+Tx3 zP87!{JOew{r1EnoJ8qXme(F{G;3CNm&bI}#uP%{X5OnspXJc!(wg+DVa2}DMg~?^c zR$%wnSJ)S$f)$b4j#s)fJ6#)FV#hkN8>-M z_246;sfmZtFs!p5KMATFw5uG%`$3h%c9p|;XsW>XBQ*_9ghDk}1f>iWOrnC2)E-7K z3sg6g5}{^&MNonZlu}L-nXOR536^?Qu@yrhxlj`av4wyIa4Q2Qf$GbgE=hrKS%Q^< zvD6AI80;J7e1iPj(c09~;+~E&Gb~*y8kYr^W#gLE#==qac-;lrgKJP1!ta=AueT1B zcRkK+VfzLIvF4uNe%CkN_bu-HE$#7aMhhxe{(wrB9(8<1h?Acx%1L@$DPF?#MT_~# zykPFLo&jY=*j5b`#BWVB>Hh&&yJy#>+U@2t`G79TYJz(0GviQFud`vW5+}vFO5r)_ z>c3axq&5)AN|v$t>TfW}aDeJmsDsyT<&I-#$B_nPu3_E!J&`EF!UKiTF(x@H&WY3* zVx*1&x>}Gz!#8MCL8$MR1E7KC>Or`T=JC)3T64PsY=L}?;UG1xyC%R#&i@hc`TgC$ zzx(Z#xWD`DyYG`w$L=cMiU_=0w=E_JE(W-1WHiBIf<_-7J?tCwU}(8}^l*(-jn35x9Yj7?_Ldjm2oE?^Y(7 zVnI@5ef72UDP=@}Fv}nSb5gVrU3(R=k`$m-NjNTv6*a~?jVZsIrJai=@-9Y__hxo? zHnaPrI-h%UyMIn|yMAZ%p0zl!D?-T!M2tT8Ply)%so#zppM3e`QHUCw@mSU%$VZzH zi5yxa?Wu|#7?FEp$5_KqqC|&=8FrZFeNjl*10ZW94WG9ijzyUq!?lJA8d*i zj9qp(g7*99B*$~2cBB&U7q2w^WO~>^h^iWTA^gx2mKG2Tl4<7-iRTc9YaEZXe`d>b z7M^|6u>Lw}jZ6>isHOoejSMPlc#CGM&^1++%xIBn;?MM4(_8@2R|JJrH_+LyRGCGv z#u?Qpi0pvGx^b-OQQbs!Aq0_{2nEUsUC}HTAyjoazyLVtm(sDU7`-+9x7P6Jrmcv2~~)R*XTY=Id)kUmJSY zOWyCJbB+6GAqijNEN&eVFRF<}$0BT_n2bv3oiMXRu5f~S`$L6GOn<;Ryye*hy*+z@ z*bHI)Nh58sU{#i??NtC_Rqfsd&}%5Y8uWkD=-57rC;>lsaHzs%ndt+L^D-k2SJ7fl zghH+XbPeasdt1>zzBj8IA)J^9HiVRJ$vK&m>j~Ob)bPcF6m&d-$ea39-;^x9al3z` z-i8|p4mYbZLFB{u&$|v<>jhWs)XQyZ{I~a>B40DPF4_TK)NJxa$=A!(s2k{pV|Kz>~REGj()Z(@ckO#n;qCbu5Gi_Gglq(9wJDE1HPDeC^Qxn|> z!_`yx3jlygS!WC)b?AOjSLg>V5)lf@tJiOyP1M~8q7hJ`G20$|&MP5NRi;?#NCh3C z;=BL@70(pb@go;ZP_NCd>Sx#mH_&8}LzV78ee^*#rE%*5B=j1qe)(89X@okam1K4N zCPZ_f$Mge_$Tgf@%~ZaDbJDOun4nQSeiT2l4i#%E_4Fu?vW#8%rVN(w(-l)vpsNMV zh_>ypA)5^5v|X$N-Z)t5=2cT6sjMV%FX*hwGQCOaz`0jU!B$0$-PeHi?F;TS&a$v^ z!t$II_MZOo>rNkuBwt{$m>@h&M&tdRKFU|-mSKE(nWq5!L`UN7zC35g8aRZC>hhyW zDj`{pfN@siV)j_}W{mgX_3sI?B%I1sPu(^1WIzJ|4FELbt%ePzIT_*7*d{;6G$ZL5 z$ruJBGj;W63ajkrt-3EsJ|$e76bUN?71BtvT$96m$08>^q6z9wI1Dg8RGCDF2oU$>tpA>nfaQ zOf<^cHv3_DMr7lQHVtikGneg~_O1ryisd{b3FE2aMZ#M81wJav<{)*!SuJ4dC_R^q zE7qyA3=nst$k}hhm1~d2m9HDFA57ObY7`O2vU^gA>`z4(is0dY(MA|86$L53)?*t3t0>d5q+_5H~8z#8dRwcI_c`>GG3 z!f^F4Lz$`f?6iR4#%Q*Kcyxy`ijpLBMw3p^Xb!U1RtF%GHRJ9;&#;yjIh!9X)NH;x z#B6@wj+Q$r21t2rIiTdVr5Yi36xZmuF02u({k9&xcGaknYgeg8K zv{7eAVY@>gZMHtuNOPMjzwv^KjW_ft%(8KRP4BPi{WX2(Yg%8hqxqbZIb19!+xgzy zWgWp+G#-^g$T^K;rSK! zs;H3|SR`E)gm+f0Z$65+BXDD!lBKI)>7!#RfHz%BU!!x zbaH%5Kx5(ay2qsCB*BuT=nAj=K+%Q^=5}`wP<47A4Hv2947G}^kj!T#W5rAMy$uSd zp|_`BXYpGUJB2`k!?nLgc#Spq4^kcolqk z#|N7I+PS|p=(4ZJuCvN?OnTjv!wGMk?%K)j%riE{*+gfAkr|n2(W@45nMkl_WKMmdr|naSMk}NqSPn)z z{#pgmI6jOYKXkplVNT8HLl$Z({usm_5`=78dDYy;v{lE`#mTQ1TSTTW9!SwG=eT2z zQ`;R2!oRI?)>4m0A~VyvOPvXiR>mV!Q&-fQ&4YLt4r_R=587E61ye!$Lc9a5h_L5 zwGN5)~)3XcKr4H6or@;i+4ma3? z`vFI%vCstFL`KS%X)B{PvAg>Z(}nncPY-U-4QzMK3HHT$tt0E%A6U$2`OAUdZ~$zi zFCg6UF+kXfnO3K1UqgO*)X{E@5~ZY?$g78}?bD3?qPb>?q71d!ss}nor%VW{_SBfi zK|+wx9$JoFw_%P^-RoaV%yAyV((`&wu&g){_N-`9PmRQ%lar00nW-O#%VWWKLLzg6 z6-31ywARU7-S|U1boFgMS(QnfGeO8EL^7;-bN}F5C0VWjgq+VCc1eoh zfY|d(fOJi%;Fb2!m{utS{^&e4Tdo4HipWcdXz$+0W? z8j9AuI0)smFp}hqSf8F8U%xGI{ncqAgI!L7o2)~l&}Aomv*{KGUHmX_jmixzW~^>ceyZwWn^sOWy&Haq zMNx&jcOSXw{q6%tpK!dc0s0D^6f<_Bw~O&lGwkRV=(h|{9aZNT2eQkH>1u-h;dS(V zP6E~GjL-gtd=2VcZ8FSFgd;3V__3#53zVh#xo_D~5z z5Jej)K2l4hp)p!uu|PR4HA84xX|(SQ&MPe^%~pGgF_mRz%Bk=?7(QDR=ugSR?x$F2 z_HDZ*l?%?Q`2u}QvV2$A4b)QZejkxyP74xS)BX)G&1)|jM1(yMdm5fxkkxLM=`dX; zK^kWz;n@lik>HY~NH7zGYe7~(K^07D*;&bn1Ugs)Q)BMfB*g&5VxhMuB}NM`l+TkQedf9M;DYBrx??fDD4IG2T>K<#1tnIaqJU{;5B|{1TZoGpHvroLt7H zO)SZ^?4jt_U%k5ZdFlDpF7hgqw9H5k#VBYHAyT9Oq_f90K>SX>_{TSAuiu`c2ptoN zX(qm}Gv&PYM0PmCJQrG;LybL(K4y~B`JAW+0waF~<8VxLhUOAI$Vh==&#)~14%oPd zXz0)E3#efX`X*m^ucrdV!AL)Va;}*E^b}Jrd^iaGpa1>;p)Y><1;ww_d;9CJlP}N@ z>L+xIe*N`_25hyS9ZTH!L}w^|b}b1naQ61>MKJUiw77KKqcs8fLhbIlCSE5y9rHZW zCO})A6oq%!o|AVzIu>fwu?%JdYMLfETt_dSoxMR3%)XTezWbkyFmA1{x2(ATk--@M)5npRu(Y3yGvDxL*>p4ApRlP}P}H9HNP-AqJD zInFK$CdoyK)5*WzHV%+lnIq6$c1Cjb>^;!r%y~u=f_m;^^n8Dio;gFkhaJ;(7`2H% zJxjyQNc0mU&h0+wMe;?A+pnUz3O&>_hqZ@#-ZAQ-p0~g6yr{mu_V={19?mx&Tiq%Y z1e(j|jQjcq2#Pa$3@&lT>C{Ntg77?EV4fw##`uk7sjcud!$Q2O{TI(YOOsIIH_h05 zFdmMFgW6Cx85E2*K?^8kanGf`U%)6aIBfnZSd>Bql zFYPuB-J*WWgcMHAX$%S>goX(at|n(raA0Z#rcBON*LWiP8i17Lm=&a%%-p1d8ku%Z0rHAmeE*&v{9b8019SRMa1qY8`qI4_xOoCwVic={` zVa!Kx#tBYW+W1OX$t0}T$xKH&38fzGp*!zjT)5K+GA9s0lI>XQjFjUS@0 z+aD|_T$ZFrwO|#j6N}WnG`bt#u-Oc#2JXk`SThs{PB#vMb!>i{?w^mTyyh}(%wXWYkJ$hvh~B$3@&sF zYD5rc7r6D4Q-=p=d0gS_ToJe%myeduaZWR;gB_lnE+6&wkox<;#X2-qR@P>{cM8fm z2j9aKjCa_ZSc4SFPzJFd5GZA8tf7M^4WH~gYRj|G)FI?_;30A?Id)EY&@ebUUf3lc z+^LBpN3KgIh+=@?P!udBj?F2umF|$v?CQtTWNhDk#Zq96cKzN8^KpXqhfjvCvlmo= z2gk`N!fC2b8WBig;srGXyJy~|o%tLSMFnzp9Twhg#>2OVglbgRN}rl~)i#@uMyVaK`K4 zjP*3_{7+8nA!ZDxU*k-1mi|TYKc9mo!x=8-wMCUNQ!AKywdhQp54ysdf%eQ~)x)2# zq9BQ}l;bQ{cL;MylPbg9gg4ZhJ}g8{SqSy|oU&Bk5J_`)4v!MBkw?2vq3y}(uA0s6 zD^`%;S_mC-+6BHws>a$uv(&!$7Hks$GWI?WH!O@1o_o^%eTe?`U(vsP_4v`jaQHx- z;t@*m>Y>^#YcFPjef1UU*`9hEhW*qtL$c8zPflxt3Nt9$ZcVr;Z7XP8^RXcik`-0(gY7mF9fl^2xgnaql{x=ofAN`5dG_FKAA!R9L7B9GRhH zP{0aQXd;)*%pg-2t1w_IF+2*5A&a_?396Bzj=K|fjq4tP32F6O=j*$zS6PGidp+Ga zITNd<6T0TI%xq$Plb1N7z%qHe>AYFT@%YPl2xc(_xoY^ILQh+bE+fTUF1*p{n023M zh>GER0V8;~!Rn{^Q;imsS8Im*5yO{?WI+^_`t6%Hr{;$*A8~*J_7`5cA7myp%KJgT z;t(SMDU3xLNUJGynH z)N4C4uS~~`6UGS>?=nf52|Z#fpRjYWcy&rJCmfs+o>Lcqpx;L?{phq$Hw1viGfC=m zvxlaYTpJX28>g}N*;5m5)|>M0B-XCtm2X{Xv#NHff9o~<{Rj7LlO)f&h{u{LamB%e zHHWZq1erWjpCFg7%y9afwlerx?sh3#|)wFXm%@s_wqF;5Yj{zwwJ zE7ZlpDV8`!FZGR|vCa|K+drtiLe21WGz?ZqPN$V*+!#m?1f55iCkrY`A}dbRu%m<( zOHvr)sT2U}XmZcUZDigbyi8E!==2)rkbytm&0i zQ^0i|f$q>7Ipv&jlKPr`+yv&O8Tc}`*_Ffs2QRDP7;0k%2t#4%7*0gV3hf5yg&4wC z+T3y*3?DkT|4hGiwkPY@P`U{JQ-Y|@1cn}H17&y;eM{hnuqmoFE1b8p8%M44@fWfZ zeJpALkr!vP!zDqV>pdg!6br0}1O9tnh#`kL{I;j+ z*no`3_KSI=Y1<&(IQ9=3tXl0>@9i(lxNC!1DG{q+%Ff$RE&I{wiQ*G&BBP*>?c5SQ zKKCUq8_Vl||IjPkv9!Jv&FgxLMb@V@$7N%k69*+&Y7P9{Y5c8@NfY9pBG6WO$H_bI zl$LddBR&_j;k#lDjACS>!~m(V8(3EC!V=XM3tO#DurSu`z}z*rGJhx2H!seON0McP zt(Fh`g$YD^QtW}E<;I%Xz#^gH?09h>zDyrOQ#3d(*@SIyI84H+A|zS zWa}{A{1%Ok40+Sw9U0S|lu>(%m{g43n7l$i(Fo!3dea%ab(mf@3+Rm z;&PUe9z>3ithJJNT{A!Sf}Vu}KWR?V4!;id+&q^}8MeKIR3vOk_)1IdXtt*_XC(`g zTq1T=)EMcO6Z@)_CTOdpTj#7?dr(a`+niSiEj+UB^QKeVp1_@tZ!ppibBY^Aa+_*I zRb&dw6tyHe?eBGQ9p=6axBT*C{U)WuDh$L0iiXZGBE_Wh!L4j zHm|}u9TtEOr5RzM7+Szd=) ze5fw`%TW+{Do>e0j9KO6zI>y5_ROd z)W}t+k6k9)Hk2q(RG#f3kljQvqS_(}G}y53H6%y-R0UW=S=T1;{deYY{t#j#SE=*h zI+~Z}1f`Zj3@sfNI&F(0B@wW`GsYcx2)e=1c(~ZrAq_$CTpM6>G6$tPak$a;_>W=q zmnh&?|l-(MXU}U9Zr6#NMK-%P%!kv z;WyMDQ;_P;XTO|E_U+k=4ck(yQLJSUs)|%!C+%QS5H%&NgYjvLaH%dD=L(DK0MYHb zbk^ul7=0yQDoTm79C5}pKa7rFodYJ1ZIlMjYM`MVe#^5-Fi)+3Hl}n9LxnjN!0Uh_ zWr2${Gn`L7V(mV%SIhlVO$~24>&(oZJK_3he~Ys6ayM z6pIzvvvZhM(mBi+wJ;Ix9O%smTw?qQhq?=efkD@y-W3xmR*ycs^tA)W#IvAXuXXxJ zil(jAHpbYX_R*P608LlJ1&^_4p z$aV@74_{LEH7P5IA3m%D@c5YAZ#e2~=s9aaR9s0TY|Y9NT&N4LXc?0Q*W@A4h_uXb zqE3h(*v}9s&K(Z~Uo0hoh|XGyXsvoru$k4qUufxfU|4&P}&U48cSn`mO= znIOgfcGy5O1#1*&Z#rlG<0#*zrsAlOw6`cP2vwH00}2c#w5Z&M}b~d|5+;u*&qu6V52qP$H9Z9T}#vx z1imyZ-4%Xz@~v_5nvGIil8kVXMm5l2xBN`#zLFQ#sCP+LJvR+Mm$~BU>JnIp3F^V$ z_2Zm5uk<_Zq_ewLjLP0EB=pds_%BshCk6C-RuRV$!aes|d2Mw#;g+)XLb3UpRz)dYl#*tONH8kNJ}7WC~}0Q)>E6$|d?L%d%sv42;YWivjwT6?xtw+@|b~;@Z$6m!a zj@>VCMDmB_7Ng}Mgq%>hYgxcxQ?i6*6ZGck=|+g-H4vNTN9$nnAg7u6?AbwS*_EU4 zStZL#qH|;1c21-xw$n>tT6l_dAKrIR`ckKSxhqiJEVPCwsNC{a(4gwtn~?$^47naO zDeLrE4@$ywswt;Elv0i)C*+};hSjXw<+2uf66-uwHRBJ!!l48cI^~qX^8!BTNYj(G zeUf5F0}|O(_vD40Huk5adwD=`z-InB`L`aPsgl^&$wXm97%k|0k*z#=p3Al64GxFh zhx23{9CLl<0(wTqccH<~aO=hk_xFlS>scCijH7f(xTJy{={xHQ_cNMMXVx_2osfYUP`P0#p(PQ+F(f8;Z=9t;2~s2iP6DfF80v=Q0$>m zzK2GO@h~49iiiHx!#M?q*wj!*=#BPR^QUZ_*tFM;QLLx~>)F76ezW!F(mB;iEXpd& zw%X#-RQ?@oa&un6+@CX^5S>WCTo0A z-1Cf3&&X@sO1(9?d_WSSW+O_h+cl{m+XGHG6T-IvGbN(CD}`qWPK;)9);ZVab$>|< znpe5+QDl^D>ac66DnTfEwiUX%Bqa(~|ai)GcFVta|Id&+v*Hbsj z9k|NXJx8$P3wiwU~fu_W}2vB{rs$wz;B{rsE#|5{DUKP8thhR?q}KDuQ8^*>Is zZ=b};O1v4m7K)5y1tFOrrseU=|M_3z|9SA!pB|l0|MO(Fs$TqO8eivZa5~J#Nel^W z-hzrLkeeF^_zu<5T<6pYGAq?OCG){n(y-zc&NE0msy|O!nQBmNUSxykd8fgo@!1uM z{DkPKO~W1a`n-PgY+?sb3y53gp$SFl=mSIxYGQjgd>tkOkXyuNlt4j&m^RD8+$BzU z2jrBkP7A8f2_${AA_R3XE;Lnr!T9__Tj702$(2#D!={_cMmm3yu{kX+)F1;_^GYsa zT(Dx5v#On_$dh}l?I?!hz9G0tohixXyF@Q-zya!96>Y5XAO(|at6qew!m?WsSjlE$ zD)HP7(lonRu$)}Dwim{6Mdsz89*j#?Y8pkVQT(GYW+PTyRFC=0riqdak$g&0KQ$XI zT-H_d974#(B_&tr*iC}DoR#R=b(t~lydr?Dxd5;_d$qM%uU)i4Lnb3dGI80=sRjZ> z=W;s65?lSCO)ly_A!kJnhCdMZ)aJEv{V8;YriwnZ*^lh;oON~+#)J!oV(0G$Vn2Mq zakxVk48tV0YMm#Swab*$?Vq*sUHK0wE93or}{($*C?>$$B@^-DbedPV-*4Ko=LmfwD`218+$(ZdeCtVtm)go zp<|2TtXIsddb*x)xSryIteb)DY`XHZBiF=aMrIOC$$}Pe7tMgG!l2}Ol8VC78>3Sj zgf|^vyI%L48_UYgTH6_zP8OY=fFev=W3*@PrASe%2f;;EzAoNc7~vULY}n7NAWm?* z=Co_l%N=3N$plR~yAtXn_d~x*191=0n8fH=rHI18i|U%>Q&#c$+RI~yGC`G0Ea8<> z2}JZP!}Ht5*m)xjfci>8`rlJ5#1-TzFFDJdHOfvif3u4}VIPds^Hb40pMj)Cl+RjW zB7=a*T~KXAybXXmgHQHtODF$=HEMvZ9m(Bc-WUqE;R3TU=3_@fzeewYM65emaYN^7 zI*dUCXnX6fDl-w`1jt&oqBT2h+#WUek5e!3V#lyLf>eHMH0t%@z1${>aAWCOaa=8N z=7kSNLAYTjAH;G2t7+dfKO~@`Np6X3JMY|wj~3L{bhlkRTe}X;P__ty3V{`|Bhd&K zpc0bkf#wX(J>PQO2>$OX>Y#i54q~*n;MT!u`<)i2z2>K$Wr#e()1BI^ATU6+Q0dQP zg2vjARDGDy0%!DRLy9|>i{Lbu%Ool}liHwbXWP_^ilztMK#k-S8=YXI3vLt9T||1O zy~xeZ)D*BO76kKT@#;2e!Vl^(LA_CL$KBlXsw{03s)Af0jo3?EkWB1pag{a0u}2r! z*qqKeWc&eTNtOl;L659>ir{N7%~#~=jr#jk6`ijNiLV1Jx)F(WkehYPNlSlsVl~K` zW29oW0yoTuxQ6S2&oW7Gt!8kM2OoHWxAZM!P7)z2Ra4s=j)8kYZS7|rm8dvwR`Y#j ztuF2~RQn)QQS;)Oul4$y_%2>%*Lu9o_3opJXO2g1#e(-T7e5(384kQ&0^>a}9drV8 zT3GR->&0cM+#lAb*u;b~;rRFJ1hig$sQRBEh|&RfoHQE*jm{Fn0d6h)dC+WvMu%dn z1KVcQZauLz*mIqid%;rYyb3ZZQl#joif9A%*4C)4cnPg0==AmZ8xLTce{|9BPFbq< z+}jjS2l|&91dz;?AObB2PBZo3+Gv|Uv`mUg`h$~E)*w9M-s|qQCHq{x58^j5oDuS5 zpNz(7@?|n6cr+e8o{gShGCm~9qv_)(`v;GPhfjus@^bD$8F(>M;yLTT7#|)+FUF4_ znc?X<)WbIc=tKt_y#}Kb60r*#Y8dT7t$wN{u5hub=0Rv_aPVU|U!cT=63tBLe!5MH{8 zDs5OreW^Wb0N>CgQE(%7^$o273i?w`HWHaQfv6DKz*KQT0M^0a1gEcyY~{7 zTZ55;FX%fHJ^ zq+VVFW@%HZ+s!_8;cD#;i)yOHEx1UXo14M$xe53UM*yn`CQgozvJ|I% zAGeES{dnkjtM`v%{dib|`vh^U4~I3Yw_O-dgy3@!NBn#LQyn)PUI}fLrYme6*(ak@ zj`=F`*?EZPGm6It@i4+!Nee5MvKjRYbZ#;V)GAwAG#3B8%>3LLELrN|SZAAb14Fgf z-ZHth#hP!92lc$^MJSrPf9l9-h|Okysf!c$&jB>l|Bn^wwWPK-66^(;8^uaUlH2T| z7upGruXOfCmk|sI(d$(VAmVy+)cssjN#Z&B_1B(dFKUy%Zu7fg<6&&vKh$Y=*Dz;u z;QcqsUq#ed`*4N*R!htW2~HNIv%MJ{%ZX!YJFR*fcDkeRv$54x-}O=b&e**9mLvN1 zde+dQb+k;gFk5$Ph$JuN>X_CsD4GY+pJ@GpZ(2EhcKxU#aLgphJvS<2=ifSw2xy6v zOwhspWAB@mNO{$FLP4j|E8s?)UtdnOF%s~GK)_GWj@8Zg;{aoyk7PcP-@lu8qHY8K zCr#Ck0tQmLj;Ns~&Z3CXOFCbGoUQgPHCUbSm@_6JStL~)jSUz;3txxp3H*9Ce4Z6^ z2HG9-PJ(Ce1K-TP(!sKP%~7r20F(SOV=H)&A6mdB608-thfCfqGyMqRll2SZTT3C& z%hjQo&?{6|QDEQH{JXf1FX^POzeRk>hQnv*?rD8C$DYq7cx;ZXr)Sg8v&+jf@bQ#H z$mbmApE_V9;ow`DJGep~zWCysL z0Ei8atDqoUQiBdG;G>eB!-p=rl=`gegANtz%!qnW=H9^jMtl~)atv#tt$)%0==WjF z@57kihcSPzFy=q9slm+-O&P4wu|VV<)KYju8MG!GXV-d`RJ zm)>1oE;1Whr&jTFm>mOq+*M++v@Q;{5{ODdQn&oXU)z1CpZq^Q;+ znw|B;){&GrS-8OAh7PqB^@iGrt>vf)o~3boaH#cYHV}JgzD69^8#<&xmo9|2x!*;H z@5Lvj1RZbh0aZ-v15QLq68X@+=+5+Jx&i?zZ+hGftn3&^#!DWV1+nWOtHNYol&h{VU;-F^L+O9_-%{D4hp=`}&#z z>d)a5(w1ukYuDU!D>WS?Sij`jm&kOqE4p%|+DCs|&?RS6>!h?1u&%f*SSvQ$4A||v z^>OIh_*#EvA1Ta$><=t&iy+0^3Y8U_IElepGLFurX-p)Qfed{z2jwM~oN?G1+6VJNqfL|nK zb@3pXi3_>lOv;R;j%}4p;_N14IK9A2!ttD3jE3UIXeq;5l^{I)^l*2ZxeXA`uJB5r zp9yDZMl+(pf4m9L#Ki;3E|Q9K#l2E?p(wh>bnMqFLgx$FS`&4<#04Z$NX;h!qL~X_ z5`4PiRB>eboB824-Lz2Q^7xlMXMFYX7D>auc#z|=BOh+5S6Y9DK4WeC@AQ;t#B#sEf7lJSY~KAAZ@I;PEW%-Pa^icE;htppup|S& zM>BU$4E=FbuYO7?>pf-EtxsLa%i2Y&9SoswLvQeP0By`NC*_8yA|*@rOzeskIyr?9 zZN`Xu#wrN)VGWZ>Bk8r;%F!?$9gO0Gcsy=>*>8P0s7I>PsB}&+u9dnFQdx&UH_kD& zUkS)>O@83EPfXi=P4`<=8F-*pg;V8fMGzPQn=p$hN>>HWX##-(rda4`B)V@u=$}yl zq0|q;WP4*XCxnLK)532$ipS$tjME8>ZwECTXxv9y9#=R!mpHlH5Hzac$qRTmPte(6*$TMz2y#&|}f;rn*9qgScdC_(c3pCZ?XXj)+Hp;N zUr}Du0Czx$zl+8yNr(K@9>eH?##>HDtTWuFBqMVj$18Bc8>pX*)1S3U8#49QJ$$UU zDm1~-s&{x_%o5&7^KKArTOsX3^v{X08}$>fS=hzA*{#|6+QQ$1`p?Dt-CLagvpNoa zT>c|roE#S-h6vb%nh=3hD6FH_}1PxRE^#pNn3M7!1DnpTIuKU(RYGPwo|<-JR3|Nt zsNZiyZxi<8rdU%KnJ~$NgI}!AEJbQmY z{^6cgzGszx8!jZaf`dRsBGA;hy(Bf?bjqV(iY%PNmEXciS^gaJ@&p&*={DV$&WO#>DoAv2- zzCk}8p9#;fm6BjI93MPs(a`nR&w_{!jSKYV+>_7uD`H`i^YFHM{I0>@)sBg{MY*)S|!OXbQg)uT+##akA zES~rY6<#k1=QOoyE zrRy$l&_DCk^NI(x&4S%KaN5rAw>HS?+F7iY49yqKMN}n`H5&+BcdKAFF#ktVwh7 zu?U*q=K1yJyPm9B^AvyR$9S9P_){t&@t~CKp>T8CxqGQ(Qy0 zJfzP4c?`z@x^nq_w2aCf6_ZIE>Hr0#{_)Jz?Ft8=H-i@Qa|llF(w!F7bvMbOzNDC6 z(PAx0((j|^j3-2o_ZW-ClwqEtr#h!{>OSbL&2fpyk`#!}kUv~hAA7XG>Ku~7WsI;Q zw^>I=!e6vDapxUH)m`M4(kJoORoX=8l`IWdP9S&T1=tN>>}o8j_R8Yt>8hD%+JyuZ z5*MjntvqK1g5ZkkIjchi+KljSWdpLhW}AaitRp}n>-UjM5g(x_N?4{@?6^*25JMjG zNXt7Ta-C#VO1}DEg9j;D-jtjrH)17jmY5GRIvu1exg>lL|Ji*pxj_c;pC3N>)6L-F z=FVm!lB<$<9X-I6WHWIyrNu0l7daKl&9t1|=qU5!|NBNCu9>(=$+XgcFG~r3$-JD2 zn^`4DBD0%CH77Eg&cqEB?8&1?!<+o4id5vL5HoS3W9P5*l+^!b;-(amadN}UXsaC|NdhcA@$bB$^NB|1Gn6>tNa z1Y@nD5#_TsgQDXB2bo_L20O|Kmo_$|Gusg%nf=;qQiq5Z#`sncwEN31NRqtF)F69S zLu0pDC@LnNRavIirH5kl>#uI_cfAyIifu5p0t#Mt^vXS#EK6a0vt}3RAK#ZsUj;^29uA5W<2MTV@c*Kt4^!A zB4-k8YJj|4H>!6avI~or!+8IRc~sO2Du$;7<=mSW=cvFEQijJCsgl9y2~4l$Y~C!dIna1DT0Izr37B2`dzfV?vzEOuVGUcN*FR znSZ=_aX#8NUHfZ(MWzeJE{)}}TKe(fP`#;)r`j~@it$VJ#zkxv$pwj~>{^w7^S5WH zCAW&j1HDbng&VScau36vjfV_cAAtvZKrr91k znHP)``}gbOMD2W*DRRi6qIMCA6oNX_ylQ+ROp&^a6)yzpsdL%eQ`8IU>GnOuBFzZ8TF_+SXh$G}B(pY!1o}58%Vg9vif3bD=$ngHg|=9t zB;7+M@1W%Bx^l+A)#z2uq-DIV&hO1L4v6+t^y z)NbTJ$LN2s3T{&+OrBhQ50*3~+8T^q6}D@w)5i;fQ^376C#vHWtI!n{i!f1nQ7}zR z1X@dk`J5$bto|xF`!S|$pjKgmPB}SW70GQCqm*PslBhIC_FKj-&#NRMBqhfE%t@op zn|2Uv`XfnX@L4yhpO7^|g_`85UqkzDwFx8reS$_l7ryD$3R#B6>!nxZY^yB2yec9^ zK3aqcrqhAt7Fc>r1+Pj~2 zLI;w0XJSFG_wd=zlhB`HF2WXse+ z|0%3*a6PBCc-~tQ)?5-ljdXj!-bg#pF|tyV@RQl)lT-KBp>DY=S|@u&+sR+iRx;T7 zK=%>&|4C+YU8^^n;A89Kd&-LBQ+R% zyhYc9YZD_iJI%@agL%PlI>i|-62j5RY1cA$t#VJb``WbYb#RI*R!M@$wZ=n8ay*;SM3n$550OY9Zl*r<(Ul^6oY{KU zz%8sd8k9Y`Pc?vGe0Dr)Rs8Sv2=3WdC#QSJU$L%!6_82YAlxM?qXi0!%~Tnk`eiViuAXLC2rx~Skat?>yIe1A*{s-jt&TyRgI=&XdJf#SYx!0Xl(G3Nm*|0^LLRD% z_M>2hDwunMO!ZqgpO`j57wYNM&=eIlWr9i&j}p!q*WThjq!(mKvWYJ3rT|+D2?+?B zO9>zA$BvUZ6_T$e=%)&=V#>UdQjLkJGLDO6LHHo2s(X?~9b(tW2?-)I0FP>;PV<0p zSsKy~%_zyVT{wEGxW9OXbEgQbQ3Re(6jl4>VO&$}Vu1tfT2@(Rk=DuV6_cl&2zBIt zPrK^fKe{9Z5klvdZSG-aN>5ecZB)CFhP+vDA{H!5CurZJWT~V$J0=-kIhn@(uwGTd zDNCEu4p{#RVt#p0>%D0SNdmn9PU-rYaa;Uxrgr#IXQm~6T6HMUzgi7k2PBzhhr^fDeYZWeT93M0kD#^3g_{JGimn6Nr!d(GQ&hi^ zHK+7IRLKHifsX!uj-LJP{GpcbB$bfrD!)3ZYa}?M346Hb`kImit1e7Fr>M+uL0oUs z6~Ij3Wdc)}s?ZL-(1SweyNVuodUAZG8MdZ0R)CC4wKl|KbM+TY$XA-QY|B9mj?ff_ zB4~uBGt?u3c^!BpMe9|Eo2d#Q(6lcLWkriAQ^lbu$93|CWmZq;HVRe+EX5T2=N+}} z>Fq#LHvljZF49Q4?plMai<(NZOQP7FOS0;Dr_R0*6V!Y5Q-w3X3?w8I)PukM0y3MC zL{3oe6|=!qdhJKw963d^77Ej9X}F;*$~K4c-s^**U*NgbvbUDTj-(-aQgx4Pk@p?B zsA(L8M`{=bV)NZC(lHJGpyC>p)^R5DzA%dr55zLGhu8jZGuZYr5Tmf zKZ6LYCH&uN_m5t@@Rsmoru)+h7KP5#BcesftdxjVz#BVZ9WCbQ==207oTOU8Ct5Pz z&62&^+7Hw@>+OO&n1~Im0xjkW^+d&-7S|DI^UYHY(MIQ_Xi)m2_#ocDJE3of(m`7R zn*_=q?07$vDCW-?ps_%A1=CmS`6HXQIMGhl(F|W^hB7>wNy4Agf{KM&xktk_47UNk ztp@2mbVz3OZ9ZBTw8#!K-u7$OeEaJ)ZrmuOh-aa?j55XL)EEFu$?3F`HuJeQc&@e2 zf=G3oz4MFSEU2?l6I#wyfR5AN3b{ZPO%++JH6L^HBSv2=NAW3;`oZXe+x+qI&RRt7 zV{A3xJ0k3$$sO^vw{v$>S^Ni^qEoH%{oIrnoeGI<@m_s~-7BS*!kA0_$forn-Kje} z2le!Su<7Ksz5Y#}mhfTeo}t#?w+wx$la71qd04106118tkm`>pz+k!g;vwwg&(Z>Ol@!S08RwTsE`li zof?%>f$~aLV1)#}9)d0nH=B}TRgWV^=lV5pb{x1sBroO4*ga^9$oLCcGp2yB1f1c_ zCw#*V;~3+53vAzN9g+<#5gCVZIIQgzE0}zzRePXj>@U#{YAk|%X#0KioMw`6Z#t^W zOr7e6&y$k|8y49SMz#y=nQMZrRgsWJmtiH%jkT)OeXfNf7r0uMuTi~S)z2`%VjH?VS z=27r{7zaP~>XUTxWccL336(X#hxLd)^>AW$ZaTPM7^_Q^5N^Z-_3WTPxd;fuC}<5e zzGEl=6JZ73Mzbfc{EGES3{0!KOq0}k&+w9QyRoK6q-orM`sgF{6znHmAeb+F*(I1c zJB4z5`4~u&wzVNiZGX@XrrgDY zH4H}D#v-58L2qXZvyeL(gqSPlnSl(Q8AxaXGBhS2?_&JnFZxF{_h`kD35+~C%{%UD z*kL1Vb(?eCZp!h=%s4Feyw!Z8ZMd?n1M!PSvw-l932K1w~j`eLqPMEam9Ay2qAPBQryvqVuJz28a4aYD)u$%^*gB6JB=lb z7y7yX4)_jHp&g1MxD3y)I2uL&fR!(H2iJF7w7~7b`+)9i3zBs=J9~C?{PJ0xr=1j3 z@WdL%#^+p65KXT({eMT_D>MIw>s4q5s$OohqIMAt=?ZJv;dLP8}H}`e_V+Juj4`OJ&ci62puItXP+ zTI5-=q?{FbozIaJOP8|}eRp(ra`g3!XA^y2cU&<--@N|j=>+xmhx+~fL^fW(jeQsF zRJUE%JJ@u+uRWKRH+R@^ZEwT%;qAAyuXoFCt7Wsb&R(n2R%?@;)_d7#eNg)>&3QF7 z*awNk)L3@dP<7i+tx?5y+D&!q;Mdtpy`!zvN3mCcd$QR+Y8(CTt-9SVy{@m* zb^2VVOIQ*vX##>sh_235$DU*B8r9bhaxG6;dgR)5M_xq66k0t>q(})@Z-2=OLDdjc z7eHn6%rsCs{pt}&t<7q?scg>gss9_)e3uhXm19$JB7$K zhPV){AU>$r={VgdQ7ba_?a-Jx@~T87;SgY=aJgKZgs%=|0c-)}rdqQx2`(%F5Tj$y zY$quzEmHxaM;`}WFC*z$O=~%l^4#dggn7U1APk!GmbZp80b)ndT)>+!?tUK?DexCo z_2f1B&V;aNRkh1t$Osc2LSF@vdaGg&f%RX?@%53e^8yRBhj6t}mzsOWI5or8#2bVKAgRvFg%we!*gXhcLQ*(t8z8v;d`Kz0@UXra zoe>?u?3AT3I%Ps519SFNtKRF`3{*VY-9-KKjA+qRZx8zKDa6yWCQ`cRmnX~SA^URd%;~m%*zt353_uhHv>tk;aB~DYHEzw|*1>>CW;BxcOoy|RV zK0%}j4eiht5*zL@Gkt_7He4KY^}9UB)_Q-f@f|z#UB~X`c6M0HDBgfk4-zybQG)HE z13c`gPN#R$7C$ppzR>-KI(8`0UIGg|eRf_?+ zBs`hT_qs|ORwI#Qm!vQ?7bMGLu^1T3ym&dXWBx$?5PAo;wKnl=AeJnv^4k0W!e@V{ zzj>Aitp-Sk-0W!f5(;$NI_f(P-Zis@VF<-_p%dS*`8*3GDiHq-tFw)AXEySRNsz}u z%%&N^vf@Obl!%1Wsd@}Z#;(u<63=5Ts@X$7RfMnh^udKls+{Bsio$+-=FB~;!*PL$ zmLgnT&s}1*4;a`pC4$ZipGJtFZ>=EJTdsa6s0cQYm{=dH|<))3DAHqJuUGOke`)!dD%*eWSm+U}=2Q5=FSaV^SnvrV0rU4GWa zbR8CMTcSKScYTia3vUgVs7Tn7@D*YOJo^dezLW;6oD}Du<4a=6ga1^h!^Yx++CM}Q z#MS>QErj69KVQswnZy?|DdRu=vf1F^S4$bVp~Qds#VN&p_2rG)O!Wt~Rjx90?~yHv zYGH+sFm*l+RqeqkJU|N55nz5LL&*XcbAoOiB@h?Ez!cI&KUPAbgq15zd|)BTe}CxS z?DNiSur(EVi`k+@AfK7lS(n|}ttMq|%oWZMtgYYbQ=4MKkmhHnPj#3%FKl$3&_Nwx z_i2Jg!zaU$pW;G^g!J?lNCbC`4m<;4YEcApLIuLucL2WNl#U( zJ9CQ+5*g5u)WRl^LTcvsDnn7QqAa9@O>QMEH9;sF#tF>m%L83(MPWcNkw?)j0DTZ-Ngc3kt}Q8 z%uQ)~&^kMvyC*`lEk&{*YS-bI=pQ~K?}FwVGC&&wgDUX{nHk>k2dHXPE^6s3p~zw? zBxNXn^Wxl^MCh)(aXSzbo#8&Py7p7yJ1aOvuUI4zqvzZ7+wY^-GlK;>sKSJC!o<5o zc$b;*uBkT{yS`I`IpOg5JqIm~9nX8)fOH#wSn%%-w`p+EdaNBj#}K{-AU5A<=SHRJ z2jN*X*7wysvF?GaBxHQ^U2d9Jo~G8nckKN8d$a=^@huSlyW;j+96o;c zejfF@HD=~LO&JxE6o?i6BI=|3Wcb80%w*v^VDqkk%)x1yp=ImzKOR0FZ5)pkL#`c= zm5=_NOdZJK@77CT%kzAvpsZb9INtVk#U#CZJkmdq$I3hL4%>Py`4MV2_!A)%x&rGG zCv?zkH^Y@&m^|}VwoumB3I&FXr_F*8ik}rOfHPhPXIvBctWZxi*ZO&!W%bH~t}-#@ zBt<<=aJuTbgpAH0PA*6;NVX)-&TiW)2iT1i3GoUeTv8|Rb;=9(WjPx3%?6*B#Jthf zG^1kiipeuI#3PF>nv${;6QUm%(lIWg;>Kn~>0Yen_DM9}}MF0A)=-S>LOt74Z^N(+ zy=-2x#I&eG6SUJ{JRT0$KG@n7xR?XE33k3DBwex2e`$@G{d;N*4jM1~DdEO!U~7_d zo5!{UGXa46sUF?{&^taj^=jPd=^1rDJw_#Ya*8-S5qIqQmN)4S{rG%df~!DHQL+@J zEj%tL5>A%Ao~y03)V^w=+e+{`ZIJAh(qW_=0e_e0e$Pz~3{;jL1kM_Kl(jrJ5o#>tP^QnDEG;KQ?OYZbccSZq; zqKF7YAwW^G+SY#NGWOiwT*-cF)wxhOk(6vF#1C5ps#cv>t@8>~Sygny?cEWAP;MJ- zxh#n*4CedcC~UK}?@2)mb#ID%Kmb?nmbj1$Zh5;*Sw$z(sSuM~h7{j~w}CKk9+R0K zTu7&dDYFw3$+fE1ox)05el!EE5$v?p7^HJHIhzrtIG=%c-jv*^5kgaj)#Qxp`nNO+ zyU#s0!jvIq)`|X1MA_@JJCnbH+|ObT+6F|dL#4T~v`&?qe8F=EVzX)CcM zjTLwwl>2RLc;A6Tqw)@%8=;L@Q?tkw8`uOl{ z^zSE=+xe@1kAn9zGB_R1cE?a2;gT zMc-BHxhN{9NpWT%Z8O%2kXcHDX#b%TYh>GFP;$wgkGStV&S_s8gPYd}Z`#G5RncgD zD50)z4nmM9!P9P#)glB{UHp;BbEq$kvgpYm`Gi9+^isM%W+c)MuZ9(ESFx0G2c z8HxJJ%~ibP0w?L(YTDxpC%b2+Z>l`g;v1tmPp!j#ZQFmZCSsAfGpYj>J!rI{#jTju zaf;S)ihd_?iq-~u%erxVdh5M)1g9@5f|GkWz3}P!^!=nhTLb0_-3sfAkWP=#K)zPq zbC8|{g%J9N9TZ!hbHC0`;*{trg>2IDU1S|mB*>ZGXHVb)E0<>>payFT{z@SP7fEip6Q$2gw8>T5^cAD0~OV$4PG0Yh^dIbHs38ih3F1c+B)5JEbZewDL14@ondQjkx zk;!-y;{GluV?!_FF`D805{s-YeDNR^Ms>^J3CceI?|=Wlda3{SzyC)E{NclHn<_^n z+;~Q2nMBN*MP(b}^yCc8IG^dO{Qi-mN<1<)Tjc|jpImAJf+r;8rjU83OolbgFQFA3 zbLG^E3}EESx&P7WWDf!NjR4@+JHe^VJnH@*=Z*uyiRAr8nBHLSs$W1XvuiAvAgxj{ z1NZC=_4}0!Q+Rmyhr#~-AiKI8II%v)lgr?eJbJbB^|1eH=j%O-xN{3l;@Lz}!wrI#Li#Z1ccS z1ri|>6^lb6!QL0!jvG?}$YmsQq;YniZ(TcY25k9=37g#rxPL6T0!3McI5rvr!~5T}<6nGGH> zVgwS#r+tQ)t@W~o?@rDcrix={r_W48agFmljwEke=J5H*^Y~iYCp_Y|!Jua1v3tVH z0KF?1RmW(JaaeGoCFho6G`!TSJRy+Us=zb0H?DX2+=!+uOz|TgoxeKcXG@8qq==hw zO*0H3R~dR@>v;A+_sI<&*}V1*9$EL|i}w%-J}-b<|qrfgRpQW?oZ$NlSiLM5mR32I=RN7*CmhLQ6vc?^xH z;WSyt)0Q$X6647ck;|Y!yTQR;MJT5ZIn(bWW4e?Sd$OfJLgi%Xa8>$qrNbvrs&sf? zohhAfZ`VeE5-kJN`H${GYyr-{WS$g-d(}vdya`Pt zyW#HhS(sCmsv=suDw7>t}NfW zqGhv2n8kmUxBK<3aY+h)gv3qb6Q;K&frVuJy+TGwlhMx^$pTnO`8M#Nxn7?IkCsi` zto{BC9tHeIpNDAb-{di6KgKv_WXdE7FYTBoeNNn#DX_Ny;73;ft*Gu2V~`$5=%2AQ z4a!(G0=EvOiPn-{DTRjjFU+x(6!2D>9u*q@hNMp@pX!x2|181b5~KBm@G3Q}SKz(} zzhrN`bD=i($f7RUK3564AjGd8ygJ8bMq5H}KpO@#l>1~09{$hfsQ z*%s-`5V=Y98U;@JY9Ot|jE;SF3JXBrGKjV~Zd07hjD(3w6T&AO+76R~ z7&i3w#2eAkZG55DRu1G%oV+V=@jXsvGUmH|x4A3o*-Ao)?Cc;u)gSKE_GBo*fmYS4 zrj6!3|f0^SJ_sWN#< zW$F^MU-e)6B2L2Mb7@UtP7v9#}i?qxYQpN78s= z9nHy-C)R=1+`r~a!tG$^YoVgy(Ys;SDHD`ge-7s+DLp7xOKD8j%j81jlU7iM)-cXB^v#K-h9O`Z^LeFZjOPg}aC%13biQn$%N;WdR#V5C_nF|$UKb-L#3#8Q2kai098>xLpka=d(1-GMfdu36p2Kx`Qj~v^QB!#ypSItz%Vc=v{^6-nxhxel2AVc~j2OM8}@v9@#e3Li9Q#jB(9?dCM9(p9DyH9jajPOift5o{E=tm+Gy`2Wj%mhR@(-S!GwM^2ctu&m#1+$m$#}4ho;TjK%1eu zqjeBOXap`3GJ-y_tJ_JD5vGxR#R|u4&T^Ro2#yKw7$qn~W1P}FOi)62TsYO#85;q2 zD&r0X!Uq!)(56%#yPq#EM1bIni=X99I>R*V17fs=4=_#UPI^TyQG%~=;(CjZu#uDu z#a2Yegj~L2b&gP+PKX5yQlXZ@MK%C?36+l6pOYCi$a3(kmdaV4oe$?cs$sEhnlBpa zQT|!YqOeUAq}lseI>_djFT}gIuWTil5nATR2uyHB!!%-Bifgf+6fnnZ83Xj1JB>|< zSKp%*tOH@dKyRgThmD83$gs0Y@?sFNi5(R2Wk1Zv)0pBho#%KEW^sQ^(rbSA6kyyg z+E%qpZ}dG@NEd+wfWhd*k;$+t%BrPw+OHgmE&@vCOAF!ZfV3dVAy>#JOoWXV;Wr zWi%1kErY+xS0}2kE{DH19b4z|Z^+|s9uIy=Ux0-tBgAVXVm*|qpLdG&N$LCftG76e zEK9(7p74$w4Fo7tGY0!NCc&b=-6UMahEA=y*H|UfzabTp^ zICI&&m{!dkK=4l;eDcnnZq{a}2G0c^{w@zEVH%<+ET$s@G%eg5?A);1&z3ORB&9Gy zcSreLfpQrjnLS=)v_=0%<;Au651Bz%u!wg4Tb=24Z#dLxaz#?youST(>n7dRHJ=`o zDn30J&OG)7=#1OVgP9IYhW5-hX`%btSlTkK7)S?&nJpjF9W;e1?|@A?_4zZKwI9h4 zIq>BioCPR0P+Fm#)cI_Q_Ka|8mnu55D#RTjUXzGN$Yw_v_QntC825fMOFFBq(51b2 z8jH&IkI=#JyV&Tb&oGcr0-a@(LJZRxJDz3q_IYgDkk4=w&u2&IHS87X^ifVTD~Uid zr`en;|NS#8V%fx+w~|4KHsnQ+>M(QtDVbrl6;*$9@@QGw*fl&po!Y+Ax=inLR8fa# zz~~$PMEnsXD^I?58S%pe@Hq9?7XdI(nb!>uWmv37<|A>lc9x zx@5mrC^g5Wn00nZ9UfO^TsNAO!l3~D8yUG3L39A&^1#=fIrl1%Df$V`mP*zESmA%+j__;|zWRMMhHP@U9bj#NZ4Mk$VC|D#HbG z;9?-X$7bQJ&SHjz|9qUpRE4sj1dVbWUZHt*nTHWB9495=c^^jCvEtLOV|-&XYCJV4 z&sPABZ94HtN>H4R$t(+LY~(8{PX@XqF_-J|4jBz_D_xiy-pinAUkj!|M$`?pX=~1Z zP=R|RR8Vjrw@?+QdVHH+#RTG6p@C1D#A$>$v(FQzZtI}Cbc%2FMukXH!B?|_248Kf zkTs@eNW_2N-Fx~CaQJXQ5vJov`%deKzOXjZ>X@XUK{4Vfn0>N%2LG&C1k*2MbivUY zBzI*!(44;xep5tZi#tpKx9iez>m6qlEk&S7At2aZ))y0SH$j!g+^k1^o{0rKti_Aa zgo1M_;B*z%0y+9*D?65p4SoBxQ6KDfb)KJ45bv8`R( zN$&vd*hKoTXUDRw03DM|pOfIB&;-ip(TyPo4 z#mE_2OfGVK8I$y*=L;!(Z^L{%^}Ux8jSG64wQ2XCB*j~v35AhHIT4Fz$dD|oD?^Gf zl%3-Y)0hY5;QlJx5IB}yax#I`yub!>H&h9NPGH!S3zUWQhyh@iMS(FymIb8{_}u(_ zImQNh%WrkE@)dNeEItQJJY!n<__|&tLaUF-K$l^2&uXu7xDMJ>490aAIXLVbK zVY&@eJw`N(@@xX0zzjTb=C5vyb~GX#z9+#sea#(g9To;oCnypff{jdUzuaHQNEy&T zykL`KkWIE_99GCM{HxhZakT!V30-64?itej&Z<~0)oS5uxUkw;71JWyZ0227$+XB; z+wd;cbFl#H;v5Fg&bG+$F-ePg;=LdGA{@_=C=^8aI$-GW6qpR0;&i8YrqV4Y zx7rlIwW9k3*NkWNoA-z3%JJMS@&}JJUJkD0AXgFNwOT%8?rm1V3B0OhL&r5=D^Qjp zlvr3;)Uw&K0Ldpgk{xOmAg%BsfN6D^0TU(r=KX$fas@Kh(u#Y*$?cG_H4Cyw_kfh` z6X9e$s}Aw27yUw~Y(=dC`fA>&23J;X@!tuuv{$8h-P$2b*#Vgl;TcIOeoxVqBoWxp z^tmyWO3x5dVTQ-;amHjeBdLyqsltQVCJm7hM=~WT$_b(Tw8Bv5aRiz*0C^clXm>aq z2Cj&D<@!|f3>5YRsK`|F^tkvgC-dwGI%U3oL(;c|P+{+Be0Wzd%3?tpDgv=>Bg*oO%g$!fHW6aYY-by@L3Selq0h_C_jZrwhx(RdG*?@RCvae|8p7s4a zQKTCafxop*Hb*U|Tuh$Ba+}%2g#0`|LYL#re6J!8+PTM6+>DRbYc7}Lf<>{LguJB~ zJU)HN6G-qaX$i-j^D=xk0&m!jd)@hg>CzQ1Ol@wYh8iI4cHQ@ z2P+%6*$8fEAZh#UBi>BT$e!HS8+p%w=_JR|G7nEog?%vGGNW6C0=RXTT-$A{xtNV7&tk0}O#G^dSZyjAd#6#Q( zcK3r}KTNVX#b(?aenl%DCr*D6$qNipOWou~6!;kyBoo)ovC=%vU^u5!oKlehT;#k5 zBk)tp2v^}t6++&d9)F%ip|bCX#Lz$;@Ur7_7Jc2-iHrLK#zJnuG@v6h%UGhcz)YM0lK^U z@Ii^XYh761-v$6Y(0c^k-5qtVPQyzi<>5#O!>!3tQ*2~lMe4EaOr(rOK*|XvW0D-9 z^Jk~(y=AUpGy`FePI;;gdp)X5O)4s1hxuR}j;FW+jK?OXg7QNyHMiq_5H#J6kzf2I zjcLn+7F$It4_gF0T|s8paRfq}8O~=hw?N%SQJf>1W1cHRMv;=HL?XVq3hw0Cw%c8? zGu)CK(&3q=%M}q_Gg0&i_1IqM-{6tqOO}sf$`IDevtz)L=hK|bFQ;d(3-b<_*&%G2 zR5^p{q!A)e0We}BDsCv709ry#lMvm5I)$G->1g7M6*kHVEQ z{lI-W@b@Goxo}{B^z}S;Wo_iIOZ9?x(=!&zo3Ym`e7PA4QEW8R<)IxdRGso@5TEa4CF zx8Qc5}cXw!PpN~)9W}pzTgWT zse-U|0sVMBp=-&s54q6XhDz2CGoe9EUSa_Q77z zZo6@+l2U4wj0xjU-=f`M=P(%Z#EmloiPUHqC&f0{_P7~f$+$q9H98yN1OU#!{e)Z& zxDnH@#?m@j+~0`|H<*f>UIYZV;JG>-Qjw1|98;cygHMW?9M%(YjN}72jq1$G)COmK zrG%M}EE zB4%uaH~|{F#uKXrK(#k~A*o3_F`_BtiR0jow%6yZ>TKx6@n>Ilm!~!fvv)B}xKgC} z=+QkbhS`{03?Ln=7DS%#b}M0mmn*T|xD?y$QpOoIq-r801g{hqa3Z%E`r!BOI21-j zDM|ZqT5a%<1e`IMK=&DXOAbe#QSRW(m)G78=VmT!u^$u}ysCuQClh|p0fGT4p#+BY zIm@w`P$MG+6)Z$wBin6}{g}qN@_SH|En-rI1^8OCsVQNr=48^ceSc1AIdQLyEZt7< z#Fb|1KAE6UkfTrsc3XNWF}N|CU(mdjQim@vgYDCY8&jyxnLCFLLU`agl&v@9Gq>ij zoY34cmr4?H^TRdHlQ7HT^zzRM8DAADCDk>~Aw?_KmvZGUYfioA(PenLQmeC?_AMwI zys4{In9bTD1##5GLMuDZ#$?6_C`&yXdOPk8?{%CyDa2&TNVfPQTMwJOGvFjof4ThVnBD86jtdtiNit|^RQve_)$>VGfz%oGNoIMa}9I7g|t!xH^P*m zt^C$V!~DPh{XYc&D1g`3y@Ix}lkhACg-Nh84A?)0t*+{VWnJ4x;Z6`{S-@EF#Qrie zzbG+a4pDW=)&Z1%a8S_0h*fkI(F?;tnwlN1Z~|}>aq;|~JO?ERm$ijxma|S#5Uz~K zH4e~b78YQC$W;}HW(nu)wJz%38kf*`%&Kj9UQ*0ZS9BewW7aXQ8Wqvf7(Z^BMuW;y zsT>%QdlMI!%N472obK4#&%zlv$9hr{$pgd#Dne2(72D`%-TcoQ?LR5F#%-SQz^xJ< z952X5Ec9|h@T}3gy!pW=*90MLKj=JHK@^L*(0?f#&mWIpag$4kNW?DY1XE7HsI=&u zf>F1a1ZJ`IheNa}Zo{oWPreE|H^K=2Iu8?uvc4G()y=uNb0Fc(;&e`NA@3nRduMz| z&&8gyZzUtd$UMQtOOgw0_!b;&0hs_Q39}e9unfbXUJr>6PRIv^F2E;vAWZ?_^)=2% zy!Iugy=~M>;sXA^3F&ysAX@!#0@cgD*lEEW7L*b}HEakysg>T&bK0~vj@=5toXWi8 zXWk3|0Xip_Tx-f>=*qZm0@5TWGebAUNT-~ddRx;A%ad77v?>T+gG^utYBox^$om&E z;(Ude2Cy&xSA$soQ~zHxlG8saP5A`qStt+}j9pV>C0&&5SRLC=I!?#7ZQHilv2Ckk z+qP}o*mg&E^3D8#d8zlRbLyP6_g=6Il~U#>SK)I*8W*n`ZWs}rC72#;N*^<@CMW!j zpcihzIcpch-lc0B<1tA=oHl>Q_OXK25Pz$ga$xBLB`v=O$mFXCTg zI#lVdmfRmtPkRPS01cbya6H-Zox@Wf~r#HAPnI=$r2~#bS@ew*t4Eh8j z6o=PBuvWrnFA=>BAgW%Q7S(Sh)2~lQ?xtU9Oy;;sUiCt@lp(zk0gw6KT zqtPOS=*PgZUoCMV&|fMB9q1p5oyJZmHz~1p{pY$mA12^I*4fsU80NVL@?4dk!#=5y z7oqoK5{GaniYGNs`dR(z+HuR9N-l^rDyB(bY0(%)^OItQ`25v2S!LDHa(r}0r|tGQ z|B^)zr3uZ+lrTliN!Eq+q^y>S=wgN~<<)XGRH{i5;5ZkB4$OmXjGRk_KJIm~SmGxr z-1p~ym^Z(bxM>-fSCzCluL+2lQ>)(Dr~FCn92VvUl%mCm1hO0H@+N!iiqz2^oL8C%QetRsJv zSdjR}dM{za-NlIKrc)VqL~fB7?MvbXfmpL814EgHw*dnfCK( z0yo6=DvgX?@i?zXl)Hasl}H+0CJ_rXTN$K|frZXSTxYbPZe&oB4DMI0!h9qZcx$Os za{OffwSzkc)-_^iXOKtg(IL)QK%u;3o}8n83yzs1zuY^HK;F}GrimIJp2oY$0-~~} zt8n6n+fDG|h}iOuAd_FOCxZ_nEQ}?0*#VT?91N6wnGvlCV&>1J6Z0yjOD?z92LW(h zLR8XB`)^L$I94PwH1`f43Dt2`nFei6&P1@yp(P{t5cS4V{>SHMuD-hdvV9c=Eu=S+w^mr9`u%q3`(2lsz1+ z#v|l^lasAS_}qg0?%(^8^li96!yTt^gEkd|{*im+rGm4Ok-ANeDTz^eID(Lp-H%#Q62ARs>^YLQ4}mnWmH_M(&-+l3 zbP!J?wzSAJaSQv`X@xU+Js79!L$nBk@}ku*H^6zoFwN7Zx6FWK#gmK;ONoly+fNkWp->VQ z2?7yqK)ksnU?xA~0-FPH&=OB7vz@vqJKgvmZgHhK^IutsZk5Nh4xVU#P*Mvty3Ui((-ORTBoLtia9B^l^i z;pIJ%XEmJcex|ow{Xh+1>el`{=hbrRo+Ob`Hy@=Bq zoOqIdBDo|W(zw!TUP#bN)!EpeJh_G`?10ou8DTp(f)OgjXLjOtCp=KegX^!yaW=#P zCWf+ra3>#5d7+W}u|cBE_xC#Bw7maTHacNcTw~lKT@z1N=QT95!7y)4v3d?dPg3w$P@)oL1h}qN( zGeSx;9JuD!!`blWVB+%wF%gbfgIj2|bA%lnzrD^lY$AsbHp;`N|5f4zE^tl7M-`T7 z5cAT!Tm}PU7=D~oG4!)(0_~@2^i>n)7?Da9SU$xBk2riiRTE4x|byY>j3{VR|m~NZ-*t< z)yEZ!-ES(WoO{qmGeI1EtkRAI^NBe(@iBtp$SH(2j2iAFT79<>Xka-7(>lKF6FkbY z*WC(3VB6+sf@MyNRfAKpDpt{x_iMSb%R&t`rDGa?k@w!^rEGD>dN_hDMk=8sfq#Of zB9bxYW4>6?)x=T-TU86^ABGK^GI9^=gk(wZB1a6*VKT}*HgwZ$5;jDi-ZRM9H^GUs z66|6qMujEdk%uy4v=TmW$g)VO{2DE48m#O{tI;lVNvUu^;?EzpTleI&^6TwdHAFn& z0JSE{<(zKTbVwu(81P?Aw@whP+l4^62LwoJDWk{rN>n#R5KLqbN!<$jJ^ZYtWiWm4 z7%cP~Y+vXpLlzHttUX0Bcf&8J>xTiYa4im_7K+ zCK34qpJot|#ysAkQI!9oz4wnW?IP@FRRk>ofHj&VEXk6m)@0_N` zPjffy6!|fC&h<(8_|c)qq1aI z10SVS?wR{20`3yR^@d}$+r|0j_4Kvk))dVhY3&A@E6K@s~o(pZd%W89`4e%S{$4; zAu+1-1*3)rmB`T<9-;Tay^FQ`6(n5XvD~Jk?UXBA zcm0$C=}!y^O3;EVRJ3}vby0OQLp@dgm(FlRBC;fef8V@*0Ie=&Gc7f#h?MeU_Nx#6 zJcdIfP{Gl)f0QaFPLw=~nlFO`sTuQ!aQcUbVq!wD&dwJEAQMAIO_X~vIh=5+C{Z*# z(RMN35;vl1&v%c9ae(FLyxU=ME_TM{xGKeIVcxLy7*%{JzJ8!X3 zGSf8WJrl>n_OU+SaJnv6E#YR2^STBmtB9q56z?le69Vo7Tsk=hc@BtC{az15E3m}O zeREX(oCBZZUoYn>{!;(U5ixG|{a$B#OdALCq-+gh_>lfwsdNFF!y^+bPf%OzU}kI~ z#tY$w={+|P7t3k`v?@)TOtI%252nVRm4`0Cdo9pFWynxF1nw5H7k(mUZ@9DKV5ydk`b;JJm>Tc zWLM`~lOG&!gBDxcvh0ehjD|b~FrHB|Fbk9H&10)pf1 zFBwTP6FnCj+D~O(|A?iNdMxOiR$Nn_u{>GNAfITN^~;$aW*~mblCAVq1}i()(5kGQ z(?qcdbVolXSu-afQbxPv??r1`p;Zc}qVK;NnSc=4qViyMx0*_T?yi`9>!n1b!J5c} zLiol=DC!ULe29MPd0L4Nq80}C1=M>M1XJf3Ql!jWP}l($+7cTLL*%%<;3CIZQn#tW zzXpwAu7t_sHZ~sQ0U8n!X~&_Fa$$`WpE<%-A{tfT4}e5jcrj=TVoUD0LBh+hU1LPX=qCa}+e8TO3H@+FJQXNrr4Es-}sc?492~Fsj$XPSddeV2F zMG32ah!-b~zQLv*`XYQ0i<1E|*P1_fJ{-RGY4dN+oNRxF4esrI@qO^)A#4M~puEHU z*Vg?zULFnH-Q90?5eIVygK`G>^YOa=E*59qRg3dNv;Xe-FcSB7yQ%?0=wdNiaKeoFw@{ z@C3>0FL~;%WKdbijJ-Slw4yxA|JARKOCZBB9Pvs?AIV7%`3~g`6}NlyJ7d~G>iu0l zkI(n*X@{K~_XgIk_!FcY20Ki0LEcf&aY;A}DUwd0yV2d>@A>m?^C60Tv{wI7;pSub zVo`ncguzSu^x6%Vx`eQHoLJ(TxJoiT$Fri>0`N*(OCM0*Ve*4>q($S>^yj+k3- zwETiMg(-2Q$GMM=AihEp>2G6J39kUiq8h5A$C@|lF1Og8kmn(;@mfwYyZI~56f&NF z+yWyT<3(g(t*y-=UR}bx%VP3dDZD%ScW_U+A(|YK zN$GdP`&s|2CeJoTNWt&I`lNo7at?bMRzqXh3*Np5#uf>b<>znf1GA2%~X+Wk(RkTN68g44Z1=Mk}AzSVAS)Yi4i zh)4i^Q$tmvwsEU_Gd=)c8rY`u*EQ=aH9*)js#6ne6#hbt391Tiwf5IEbATza4K+hk zz9}^%*h3TdFMrh|09ef)FqjM>=ZF2cYxQkbe{4@R80>HeK%XOU@U^7r7+TkGJW1{z zDBN@({@jxk!`QH`ypjyGth5_jxyqLhNJh#Hb*b=vOiwuuhzf3t)_q!78txzgYk3zf z*zMFvVMmOEID^VaRFC5$N3-46mxih-sV$=<<@|Yg(f=fuZnEf5+DZ<&0F$51&DM^z zVCxq1U~ZBc!YS6x@7sxbi4bJyzxFVVVEr9jcarl>n^Z zXgDb?*%cQV_YZCy$|P);73n^i5F|(s3y!b!4D~WhMGv~jAviw+8`K=POOt+s)j|vb;9K|-cF6gSja28C97>>&)_t4l@hT%q zt*KOmyF+)0*XzFxW>)tVB3LyfL#FvjGG`Zmb72kSWggtT@OaqXE{`x{ zhbpl3r*CTyJ;Q52)gMgjZn60d;c+B~Db{CDClA0rE{MhJq$@iN?Xoo))KkYv=;X^5 zprXsU1WI^tJ#+ znP2s$5$d|tC&jqbIpElLu_QKoxjX2ZhTMCH`EP;jJO7%KV@}lbROO(fk&+eal4)QJ zc*a4OQ(#ugWzAu=XBA&0(sAu(@Y9=_O#X9Ysp>xg@rb{@*z$FozV{-&1oP~3ecsPI z0=$ax$m8nzVB>ufIskO@i!rr^4KkO znBEM>iJqaSIZSzM(S~NG>hP@9a~mnANM+GV4fc2#tAw*B)RHX7i5Bin7C0wyyK+@0 zZJHb_J}JR=9I}G3P{grcv{gtVU1+yif}4uMIe7Aq7NYqwOYg>rPxM}i0}0T##t}n| zzla#D-sHrJcimh93GLST^>E@{(@hc5$hU~zi?nx#zr$0E3%zN^gKX51L*aApssBpf z42G4Aw|8E65}6pvy_+~Kt&j=hanPAqRQ$z6rlvj*TaImn2 zO|A=Q0@yZ`V+|^SgotoJl&NuIx$*cjI3Obx09G)QF86LrgZSObCfQvv1IDIIKU;<5iJ1RSa1{U<4bEKEB!fe8#ep8MdY zt=RpJYqjp${hG_ithe@42<}w_ITLrx*#3JTQ5JWt_v({gV8$)oLxsA7X_PU_urz(# zFjUIJKO0h#pRKGD96hhcao)rd?)PHMQ-1Ld0#iG(x@Mmo11I zj;r7;9P++OlvTSgU0j1;+Xp2IRixQN@JJ|fjaW5N4t~=5FEbuE1shTA@>0W6YwUpx zQ1Mqg$vU0ImI|d@PS>G$A6nb0-5r=b1VdFb8O(&3olTafVk+%Lmz0TRIc?=LRXIX{ z{T6IMDB9!{DA!(X=f%d1EOtr%VFfnIiu=Gwje5<}LA~rD+GCX#HuqwuvI~ceGOu9gjveETbEA=Rt?Nf3RiS;sY^Y zWWhBZx3`odq@=sK;>^IoHSt16fiH8#3!{C)(NSzr(TK1}A6I>_RT7JGzACaD4}^V{ z%4I;Onn$75P6*pQ7Da{g>L&zF3sihg>p|1lPIhGK^cgW{`bcvQ3GbjyVBeApob3Ry z)}6@1wzBI)kKSRj?eZ>GO}aMtu*tjy=i`8DBrAuhySXceDErDjN990@`1)6IY0~J~ zHv$@Yzyl@6#r@Z@NsC*A>L=xpqTwgDVJ+faij@bC)Djn|7)q9Km0;u}eHZi3cJ_m! zR^Z^hryI3qpM{G`fkBR#(^!7B-U4csW%}H;l#ZxmuouyQ(QI%9qfV-Us$ObC`Ub#d zv+`qo!{7n* zq^`2EMf3frbr#wkM0rLEulzEwy8gWJGW%K-|2uC~I_$l()N~;f*qe5=uI=JE!$@}| z#Aa;el#zbBPFoxB$-Dc52+pR!l2D-yN=(eiZKfK0n2^r{|Hean)2Kig??Vxr8l_4YgLg3On^0XtE2VmO4caT z5N%mjb$|C4h~VAVzw)*ph|1*_pU`%Ln(yI6Pj~!oFJXha++U;DI}X4zuCGyH9zrXZ z14taEpop_-lv}(zB%6IN^B^cpX1W0OsWGE-AxkF82d%G2Z3*J2l@%KbHg@4Gn9>e8 zHpf&x|G}TmiuSx?*P3F*rD#{)v>R0*xPq@9Y13ss1lFw+6Gs#fQiq%v*6#)%e|Uxj zd@Q(j>#+qzrvuoZVR1W9CR*)j6zJMWs^|qQJl4y7e#3iu^CYpQ>+rF8F)`vvZo&&4 z?~1v*i{sNY&H%PQfl-!i-VSTcOoWoNDKq}`&UfQt!<0{rT6?*|UI2N37aL4md;<-e zjK=kejV{f_ArtbtB_+mUyfayp$}?)eS?2yqylTf!zvA?STm3 zpk!0CYQ0iQ4C9dNlk0CJqDVng!)547l0;}K*qI7-GwhpdNTUZ#FpU@t@zGRtjV4`k z3d|UA>;MaWFzDEY&APi*R?d+fjV^vccca!TBpaJkGAX8CI`&v#V%x>lRNC&`Suw=5 zG~!`zDTg&Pv@37fcd(${7k|I55<07jwlvUa6DE35Y0D1&Fx@Y>x3!AkL#1L+sW%2N zI7$#N(g5~WI!W<8ztyoYfK0 z=O1x_W3$Ix*8#RmC;l{NuER>V-%T^ymlm}4KFl=YHZMY3;VhAEB;M+$@qn?v!0yXIp=CO;j3+yXm=aV{>`k}W~Zz4)$dd-$u3KWyleN)tJLnOP3S^6 z^N4(4&J07--%*PhqiTsvXs4?+3+Bp=ZbLD}7@px>iC0@iq6i8hk*dCF|-drZl^=tMRGo0b?lX7gaGEw7`Aa$*Fg zb)szJs%U2%1Eoyc81H}Yx{47t*ezybIiht=M@XPiKStU`%4Qte1YE{IKV3a~TxrHm zpk+vC4tSy--i_uVEAjb0f(&e@7=tmEIBrUTw9_^674~N3e4Nc7&i_(^F~`oCm3L1~ zHohZ2IkUP*%RZ^lHJG+{@Zk($*%xM&t%7gQb$aDf(F%R;;eiE*e(F5)Ux7E)FLfyfKh8aZ?@4bj?Ph| z0FT#%BA=yu_(wP3EUi&_mYrjuFjbi2Qs+#GpI&)pLFHdSR5@V@&+nrFx<6nN4XfuF zxh%rPfVmroS5}!JmEoOe{UU-fGvk}2pteq#aJ_h%K6cn{a0A}O6;Uf?w!Y_v-p<5L z3R=^`5nJOZcK}K#j?AIwShWjW6BW~d_cL4?7W9MSw2@bee6TJLdGtHq*kPR25L7xoZE5}|cGURIua5$~2ZfajMBAY;FZEkBKW z;CG+_Ke4N3s#!c731leu5=ExZtlj8v1xVcQr8HUMO|Ne_$Wf|k>cQA zAGlnOV0`idBChUwn{+Nh-+^<{K*X>x@(+HWj>Zun;vEb2N^3(@)5r4LX+12BlB=1i zsH(y{^rM`aW}rF85f*}zg4}-nGc4M%_4N^Hp`d%ztc9CYD=(EbG9sC>7=PQn9m2)=AO)oe)0!S1)O_J3uW; z>pKih+VtIE3e`?@qa-O5y+`m@Lu*Xga9N}RVSS!YoJqu=+ch$T_Aiv5ZOv+FhNYyP z--KUb=PpxAqjv=z_Jo2_H;~CjSwL2b#aKBl&23SxYH~w~7XN;xvua-o?wAj_n&U-g z_^5MKYM+CkFE*^unGf4Qd&q!XNVive=Z4h$;wXELmr0ZAdK+^dA+@nWiubDiP05oT z6Yk6YP({vCFt#O$>bvCpjL}S$CR1W1Lh4g_d+JkTMJY2dZ%?OgzDUcOr^`EX-4^Ss{RDCIjQxik8X(;%&71;{fyN>e`{ZHlI?$i${g&7|GC*k?PV}aHPqGx*VJGLC|y6`med8c{i z;dM!|q+G_^g=^I!{2-a zsvUNyom}9g#3K*=z30ev0sm~#@z>Xm%YDvHPAQSoa$?~m7F>dl8scB-94aS zkBhFgQNyG^6Pl(sV=XY2r0i*6djBz9TqI4s`(KIH$s&T99Rw$`DSKkpz{ zN)`u%$!?h0Y3!k{sDbUb_K(N5oFV=fkdRiwnYH2m`-xnokqBFG<50~(8Lr^F(jmeU z%8hW6CnE4o9|jzmqQK;{s0oqriB8v zNqv14KQipso>&z;;p4%3hf!DHBLu7|+`)(sQ1s?iFdpU^JZO_DCBhJy2WY3}jffAe z3-@aV@b07B2v387n022(VdgKuHo*N3ICDLN1LPljAV2W$3`|8|U%m#r0TkubGe?c$ z=ZsiY0E*@vQ%6@F0~s7$f+-A@>b&<7#-4d7I589nk-DcHkkj|R8Kb)quv~8?(19x0}ZzRN@1h^v&q{p2rnEuYljyJ+8>#K4}tay{3y z^~t0~^XI6=q?0#MKKZyPcL34Qj zvzL(y7z%z>Mnb!CzUVdoxGqa4?9YSPG2*@O9PI1$Vm6;jVcfQ7Ea!s-{aBh;))cbT zw8XXIy)nipngSW&dJ$1B8vQ!v{I`01kM0N&5MHF;0DF5}U$a3xc>*u$H0Bj0x&|+w zD-oH7G0Rg?pMt30@-Skf;0 zKRJS0i3F(KM?cOv4B`%c*}FVc*h^*#g8f?Usp2VVBGJjPiB9M#dy%0|Za9Vp#6wqYt7d|3HbSt*M-y zHjO(9Rve9+DHjX(_k}jXQ3^fPp~pElYxY_rIeFi8!kCy{Y^mBJGYElfsSi?J%M$il zI;B0Hc8bWEkhNMmn>R0P9J{LS2nNNRVNf%BsLkRq+qcf;$;nLk?y80E>#48%iDWO# zAddUNpSZQ~bO4g=-M{Svu=BOE$O?Qxnx*~#h8V^)L0K5={pFn%%GS~D%$ z`4eA7H&?Uuqtam7r;hqx;&5VSC!MRbrCw6`anJqIPvbF~r z!~h8oH6gxU#`CkK1?+1L{v)gz$e4k>SRtkOUkLRaIXJrJLYs54EIv)dFRGrL(o6P$ zdsBCw6a6vD4#W2Q*5V0hP2)Q)DmFPcYQM#bM9&W!5)EWQ&{E#m1CAny^8*0Yog_50+et<|!(NK7wa-d^mG}EOSbL14 zoC+Ka#`x=lp4%PT=(`v^|j{QYf z5Qah!mQ10Qglq}J(K?O%3;*+TpX9K8lAhi#5rr~dTquIHoUJ{aG^Fj@LL$hwq8Sqx z=X%EyRi1v!Hgp{GtZn^uUz~!${^x<}iE+|nstrx$tGLV zmc!%pvIn8tc{k>@cm&?YPNIX9Fqf`_!x3EBTObV}=WXVKwYo)U zJ!eE6ru*rstP%E~zO2B>@`(O1W2TIA_OA&< z%%Cm>82o zka#zyqd0614flUStT!>Gl&iM1K2d3^zL;{8DhO$WMba$qFg8AUZZ=Upq5i4RD&SMe zgwhDgg~{^QgGnA%Cdbc&>^$v#!Cp)xC8ond4Oih`s%ECla?(TdBcb0c2K>qJ_s-vn zTQDMKh^KhZD=mJl_2#zzycy&nc#ErkdT9EU;CJ8MKY< zWvBOh{EOQ_@B=UoTLuEWxj}T-{Q~YfxPSk``cjQBoX=*Hj#J{_YICEkGxwx%wr85 zxZAWD9|l>`>F2+D4XH}$f9OqC_S;w3%CuA#Fg7Sxjnqt#g>|H>XnMBevuK2&;bji{6t)bxHP?l?Q(AAghMBiH1H`;HB4g>;5C%8?94TYfHEklmOU+f#w$SCLJP&HMYx-#|0RKb}(n8{ktDFB3 zd-I=t0&?LI)I0!r#tsUA>so~7ci2^rX%ahOnp3T3-AJzkBnOQY*W1{2W8U7r{QM-n z+~s=gg-rd|3W+pDk=fO1+|$)E2i$^XrL5M1&4nD?g13;w=}MXCsq^`Mi?M}OS3zkt zRrv&5pi1v9swMMR;Y}3h_W@^ z!keK>Xk}|{+l88H$dB_fc{*Q8;Xe5(sHP-GnJDQp+J(h7im_Hu7IfBGXLx~<*= zp**Qif!`!(eRV8QwUk79{7LJO6z?&q^uS2`E}m*#_%+orPuT4+98W&#Ix@TA?#sq5 ztIqh5X2OSY7;3sqT5pOtlvRLQwNzT`-J13TXD^7{kSzGdT9N(yjKhmTD{63~(r2le zFS(7K1`Y6bNff3_8qt=*QBZNtP{#Is$12m%ueEAV)9SX_;_~H@))Uyu1ZCX-1keD0 z+!wFCh!0h-U%)QgK|UDx{C@>uWS|=xiOUQJfvKif0rVE;EA0WI)iW4G8mZ*!sibOt zi+`oyyt@HIbx-Ln)FwJl;GNa!WvmM2Hxk_iMw1;eW>I2O9b6*k&Bn=Uw-dI;vb}Cz zYQn(fivZly-oftg1}c!3*P~+v$T#N9{|Lae!PSZh@at_Ud~#M&qGz{@niUw=DKECT z{Ah41w#7NKFx{vC(Amm$_IpXdEpct^IyXsNO>xsUR8+&ma0z!X9L$S~t#1ewu>RGm zg!!;GUK_EC1}94kxs2ZSf`de{u$BfdOx~v5l-~-o`qx828j}Q3Qdnh2FB_$wG9CHa zj2^W=-vHY!2G@(kGuy)Aw}Lb2V%1qm_Z%poZkN&Bl6SWC{`|}w1J@OdFH5;*4R-zR z-*F3kIRy50YTpC5HrE0A|5e~Bw?IDN?5spsBMOxNq~yl@f{zuZe>)_N!cf)Pa4*D| z+ysW$;In9%3ENd;-RttQsY)|lB*XQxK13a!_sHh<{<&JG2-|aNnx3vXM3-Cwv=k!_ z#0Mfqgu$mzqa>;Hz_;U|1IKzXt*V7IuN14RK@E-+AtTgAm4 zo#eGxm|NwsIw<|q8Rp7hdp<9QtUHm>rWK(Y;$ZuLW2^L+&-@=Gcvnm$GPVE=>3c5_ zWFt%q>-D`fGVUrp%~=F082HXhG#~+$=eB2kM1SiiTfAhaqKibG0_C7CEtd3&Z)0$> zCqCBSr^OK9pbK8}!)s$67uKY$P(p4+4NyQk25pQUOjS{L=fsdXqbC{_a$>609$tol;AwSQwXIp7kjgDSO3 z$aP$*Rx(rEzV>zT9G6ONPF)xE0lWH3ji;XgQINGmAoG^vga0M}w&sbycf5JVJ5c)N z9+~6Xy@ra#tl%&8WA*b%70u(5JNkqW;?_M9x9&b%8Ju+`e_M|NGIUQQ$glkF`k2y{&?d$vGO zRJ*wXZu*adT#jBi`_-FbMbD{;&Ym`IyvKY6Zt+AW>t3%mc*7&1l0VTT`O;C38OB|+ zC6swJ2{2LRu!mJwXmuUjuKH4kpH&|7i}2SU2(-SKH$XQEu$Niq{rjx_{_=SZ#m`L7 z_mk$;AfY)OBwrc>PB}QQ@3SHhmo*a=u6=l4BvJ;I$69b7?Y!G}{_YSCMPKf~xn&>$ zUe!@A;yrCRu=fZ2?dKLqEW+y-eE_|bfwyex9H=^&mJs|Fvn+>-h?z31l_AZMO=>J9Hnm`STvY|t%r>?256^zcDSFZZ);T+Ae-|I< z&rzhh#XM*h&t1Q-pnuWFlABHb=avgVJc88yUY>2ucOdVWvLG;Ty!bis`FM3!3-Pbp zWhAsIYAy}YJb({1C*D3qtr~N|mgO{*Z>)-~LQIw|J4cOm($k^m&Z9uXe)!g(9bWi;1)^}G2YPUzB`CV-iv z;2D!T2e%BSEyTVAd&~4zkczofxVmVRwwUJ2T1^4BSxT6eVOkwn`;er7|KxgdIBB+=#rQ<=yHRZBF%8TZ~W_@Wt)n^}4hSF)T8y(D3IZ;VM zjrP~PM!C9%#mYH2H4uFPc8&E}gtvY(EW{epCdv3&Rr$3@_0tB6>Zg!uVxIh&Ixn`3 zb9h{0miEM!aqg8VaTPNC%FAucWS45s+Z_{Y*3B!!#H!`|GP#+~oZYnwfGB!@iY$HM z&+8vH)kt57Xagy5oxw_N=uhFDz)&b}@-KUPkG5kV z!T#NqZp#!8Qs;1KL?KmyMN+*nCB*qKl}(Ty>ksc*JJp=DbLz~qe|tD5o_K~Z#wa5& zz4)D~y)bae2iWZ+ihtw+o<$x&i|#32RALtc_ZeZmmcuTjos2IWzEEW_rGSeLU+yo6 zZaVKbpFz!^qk}yAw05DE!EEdEZ1Sdozu9n9 zU>XJ+Dlgjk3ic%&)mC5HMEl-lC){ogKy+7biJ~g(GQ2WgdG8;x(i1H$e24{VfoI(k zjd&>8)6s8M;VyjIFlo7Ii55{YrcXlEUkyst!+({I`jd|otIY!z9C&=`+$d_Td%4fQ zTdM*i!pI(8hFEY}mb|c@+Uj*@SASYlmz&BqZGK*XENE3N6kZGA?`ofY%VC&GwayK@ zY!=ZO&`SAVh&jqrq^e}(*vP5I$!FJju*DoX-8_a%O7r(2+~;>BK^pr zHeh4!Ewm?9x9@Ebur^JC(QF52R~;^|ifFc!{`aU|}BIS&7&4o zXgkM96?Z~;wwljIw2~G%i7fSn30%c8k8!kOoJmA!DW1JpSWoq5W+J@ktAXIT81&H( z^Q5EDj7c|fT$k8Z)Q1yVf$|lF@}-k|r5%FFUF=E4uW}qH?T4{O@F~Xg$iPnHk2b*J zX;SjetM!BHY%2U&aJYl9=7ekar*wKG9``w-d!>dfZzHqueR1K7ogDdkG7nz{xM%qv zTsi^*0Ngs8-?&BD5;&>$5u$NxMY_wrql8Zz;QNE;xUXkNWdq#*obxj=H}paNp-x_` zch@n@MZdW8?Bgc+{u>a6UcBBQ-q$%>i6R8E`s0_}fn)L*81w=dkfF3s>p6}tm|sNs zn@@mD|C#Y2jO1%)g=q4_MxBKRdoDrXvt$FebM|7hfR_H|_`oAU0!LudmRkWbjlNSs z!VM{b^+o#z0)q~$U}|=}Pjub^Epcl{Q&%0K{60A5>8Unamx(_FfZQb5%PZ~4Juv69 zU%j6JLed*@mXulz0{L)ach&Hjn*LNw=I$pJ9ya-t)mBC5Uu@9_U9SYtLw-ONLu)h2ft{L_mw#}XPPqUmG}H9#kv>d&@Be29`#~1rArcUY>djI#|&=8QuQr?p~)Y`ho8F$cXO!LRxqTfK6;M z+T&*h)#jJx-M$7E&bpzeeN<5cZ)R@mE`i)S03hN6tq0J$s;pI5Qeu}JMCIS{q&Im@ zT_Nho2;INGky+KtL~wY=LbuTD`im(xZ;fw(giFm3N=fk}Wauc`2cqkM22z1h?#6}5 zmM{91y6VX`UwA(_?zB^3KsqCsQ_&ecXdt%`;d`O7SM9Y5o$-HSm&QrWtA#MB0B_b` zzJqB;7jvQT-EjoYx>FHLa=*DchX;O1$@g$I_WRK0q+7M#7gku=+iN^>5riv^>ZKc2 zx&A!O4fA>Gm*%8fLJh`=3#f!5(wjXPlSXd&l6|+pXO)`@U=MUFVz+-Bn#(H9mCp|E}vAziW&;9{_6J1hDvH6YU(dgmfKq<@;-wuaJx9 z)$Dn4CTO03F?<2X0n4J7#!(3uZH}(cv-PgshEfZEr3ly3(Fpe<&nn^`?8pV9c((}I z*INi=IPjpzZ2|>$hW7_(6pA26ycl(!6;%aARAovEX*8YUg%oK;#*4Ak?TE;9uw~ND z)pKW0sCd5dl0n@{zv=+%v^C#2O1KNKxm{g~4OUiMF~a*G#oviefb}Yee^>H>wa&x5 z^yks3@r37PqII+Fja(B~t=^~mwTu7p+85p>e&xg0_dznr`E0e%o9P4sA?L&*qR>hE z9_$ZRvQ^MWglHc)o+f`(;Q8~8dAUHyq65WH^!8^E4rD>lY#(RAjYFfq1Jk?L{X^gG zrwX4Jj$fDH>md`)6`lb8ZdTCflmUe;GK5wVW%Ir>U&MTx{=xg4ZRK4nS!JZbQ8>{ zA1B(OM1lV#ucJ$KKJ|N`yP>;J{P!6*f1I61k9*6sgWhd+isuJMM5iz7N_PdrxROV1 zXFpv(X0Z3Bc(i4vmWz{G!KPf4`ZqIqGACbSRwB6_S1Zu0J#Me-pwbm?5Td~^b+m@h z+sH?4A-}$bdVS0{YYMM;EG02Fi(icZArdCG=DMksR{L#Y2RNXqxCMI$Ua$-3e*eEk zyGWC!5(j;9!dCAyge{c3y6?OcMOs05|iWUtf z#f_<#PUf3JLC@HO>G&@rV6BwwkgN5S==l`#yJsiq>Alj6l}PF~OKrB;>o`X+@2x)! zTYE_}OqS5EICf>(^r&=#uDE0oDgl5y&rN3IA-h4M)5m5={q|`E_Qlor%wJqZHPCe6 zou#~X*i*yNn>{vl4*cQ#69`+Gt~a@5Cb;*B`)vK(Q=0me(0Q)Dla8SxYpt+e;8Vsr zlCQoIt6(Gz_}%pN@%YID?Ih6~7F*QdLzW*t>QXWtXr%^Hj4=d1s-6w215#}7* z4AE)u;Uhw{)sKs>ySOdmhcm)1+dk-C@0%w}q_JLAd-lX}ov)CEC7n3-}F@gwYx8z=KETd@%6 z-YaPz3gJX8&8Al#_;U^%wg6M!nyWq0+AeC{OkB@}_Eh$JZT*^G587)!jf~uKUQ`Hv z2TpqugHfL0>w5rB*}1P`E)&WcI%D3D9u}8YJ7%!z$e!T5R!9u(V*M|w(euS>7-v5|bl73k%8vXMFmcIJ#fu}|D z2sJ^eah&Dr<%upO{x7l7Zxi~0T9|56wCMq9m!L?fFYP zB3LlC6{w~}`i7PPmUTJLmVzaa0PF=Ce05 zcOM7KkE3<}djed+vqC8Gx{K{%(Oo>Ic}y~ndiMdc=tQxQ;4?X;CThL@%IYFnb}{hv z?+2Ci{!c#Jv;$6Fi*;L~wgbKUEx&b}ipXx$#z|D#Y_$`m|GM?hFx^oM4>IDRlV^;C zQG3-^rQttSA-ERw8yc{jKIHz({V~Nq{hdz!urW}5PZ6yS)!Z=^IRJ?7e36Qk zrZ0CPNq*(`8vha>0xyXpj6C`HTJgTS{9}9O@A3Iq0azX7bhDCLFw+*udYp)+9Ji!M zH_5#J`yv+ylMo-ArY$)m!*^=!kI*NE1|3+)A#;1cZOgYXB)kLCn$yuo#8UY-$|ICx zl9>SaCzfQZFFrO0XO$iat(5W8ulW_C<+p*4usdzE-Za+4jtl;fF37)X47tV@F*FML{1 z4z?A?PA_Fo_)9(Jm*#O(=wgmS&q(^@BR8qSF&>NB{a`k7R{GoXPr=?ii)P~(x*3*l z(uwtcff)7TIZZRFxgQM|GYlEd0b))oq7bh9>Tw|oBlF!$sVoG=rq}6>B>7b7FrsVIy(t{p0fo115|#8(CdYJ?Ho)=~qJX?XsjX@LzT-7 zT(n!FdVnEZ?VxsVscVnqxjXjZhw>uF5A6kT4ePh=rP__`B)XQ=0YgB0vY7fH3v5(1 z7K=KNP*aL_o2ylUjQro+j3x5VtHinY(?k~S8K(Yzo#B2R4^sJ}#58$^GZYoJF^eT8p?@q^W*kS1W_Gs6qgyT!&%wSO(Xki+?d@L<*PLeCP{*}b`BVu z(!XRpeeq`tLKd9Usah=7h3M~%GrM`j&YU4L0@;?S+t)??b8k~SoX`3%|LGm=%m5xq z?3!o|ik<{r7Vi8yayq`65TAPN4yfC|h_ek_?|?6TRwj7~=k!0<@(2gusa*}1wshB# zYPeg=fvc_32pR=tRs2vle(qz~XPZpfv-T^Kq+Pe{nwS+K_J#`l(Rv{E>ehOGlKFf9dd&KK3 z1Q{fnK0*;n26N+FGgs2f^INy8gac(MrNEbg+c0xLR8J_2$08*6C4@g?ORs+f4=(o) zUdGsMM=UTnmbW*Tz6XA&mfA_9f$1PNUpK7KWwaX=c9p0{TWI|Lb#@K0>-0oA$zk&Z zzhp;IeWCq}fIcPsAu&08oL@ev#|;9siju{KF}^JkS>V!TT0yCUl}n8O0j+E@N=P%U z@>acTc|cd^JdVuyO^ankyuhrqHG$k@(uR6_&3$BW2{aP#J%L}m4;mHf^w`6s!IDY| z5_ddH2Af-GL0A|4F+v8b@3BXbBJu7*ZGKM3CwGBva2jdtQXj0;JsUE=S4G2425_@6>cNRP6! z8HOKDTs+Oge^kxLr4oH@wdo#bPb>5#Wv(lNWX=v$*~<@9UI=*49B-myaBnplg9SL) zKWDsjBR1q|wt)!6j0)&_l8+3Y{y>4YNk4Wov%?Y4O)Se01-Y!3lx}u&iCshq5d}3e zN}I;-z*T!Jfptl@U~()gDJXf*(A!e^qA@fbIVHU;arn&B7|+6swvxB{U`Yw2K4Ykm zF3j`0sD0d|v=*r6pxI#MB&if)cMpfZohIJ0yxt`$pG;+yz^7|iyR0Z%n-G%;3yepc zP_d&Pw^p1|9urzHS(KOi7?%@f)`C2V$4?h54H6LyDAy$unw`);3V)Jk`sr#{;zx(y zT7h<0@6FVe!KZLui;xNLtV_|SB1=xxYFKzy9$Tx3>MXm>jx19K6Q2?XMPswryoG9^ z24mOzVwO$z0X{&A+?CmG>Z0hy!zm>Km8>9FpU+kI#&mlH-ZH@QrDZh^G$%LbN zg>QDj9texqGma9kl}C094_hbXDv|QBsj6)T(FMQ5!Nv9BqgJ}JEIsQo>DUWL7~UEA ztTF=&{XtnX$1~B9V~`&?%b@!e7)Bu1QD2TlAuRo_GmsHwvpmY2`0SjQbJ5W>%+81s zL`@)j10J$u5jpHNM)39FPFSPRJqizE3X@BE)l%ow#JKKJk`wcvL@Mon*_4 z81!i9Ngnj>GNnbc{Nr@3U-P~P1l*Z{#x%cyhJJsoo@jJAe%5;?)a%MA+i!ZU1K9;5 z%h#HZhGp}vm=cB@_d4yK?~`(?TWwrfsirQf>s9yMZOU6c=)$#5zbEQyVQRT+E40^C z4ik|YVKZl3&a?X&(8>k<#p)tOwX1Uo&`uLSog0e2pOg-_MbzQn)49{>6?QCs!3TBX zJ$dMJ+#kN`5rMyVCR+$>873k2@D9(WFG+GAC3Jw_%a|VOp>LA|mquUrluixQ z4bktepLm!88o%d4C%5p)v;apmWPA-g!LmrUi~+-=@_x~LKGd$?n6 zFCc!2!GLV?Ldc+O(#CY1t(@VixVZ4x5bD3Yn|o-t4XZP(=flHP{7;H;)tRw1B0ZVP zajU49zoq6O_dwv{x#ZeF$hIcv4}c=Y27w}XwJw6Os;L<%x1hD1F6jsFO2o|*u=ZYv zvwL!9*7jaJVLp;n`ojnX_544t#~+RmYihRr-d;{OpRTI6gfOi5C@T1S$AfwNbE*URKg5PQiqL41X{%#RiLp`O6wEDtm26%jBbYm082M72& zOd$#K?q>q`o$P`FfAQ`V0^=k`=`oJ_-rE~X@y zDydNf|1@dK137Rx;Pm7F2wsQ>0f<5cXx#%~fFywxrMuM!U~0Wul!3qkp%s7?0~($L z2m#t87eJZfKqUo`J1(Md4qC!qOb%XEg)$kOmhh|JOKQ)9jCynYNg%<1lK-yD?`zE z+pCw8gaS9CNN4SsYt4lIEufv*7H}XF^7L_rGW#44sW*Z!eeYl1+;|X+i2L-9f|QgS z69lPzV!%r)q#Yg`H(0j?&tW^f07OL*M1u~_FbrEpF|0I;I~fHaPkR2;o=T#`%`>yLt9nA3o0h@sDQGeX;whlwy`+f1+c>@$@gB6S`w`R!u<{DOJ$cirn;3N>EJwW6D{9?VIM&1E8D07mH-qV`kxo;%I z0QcDj)8nZY#(>|}Ay-VryP%A>k#WLSN&Iys~rUNYbE&7uTC+^Y(!(|Bdd zwX)bfOTzLjT+Vqtd)Z{-l@glhhC0E4iZeY)EnCB1u(N|zK2EIR{r)U1CF);>Y2Oa44HAlQI|HOGfF-<{Nv z%qkip97`u5eU29S_-VB29^;1bKr<2Rrvg*qQtCF4?2{f@a3b>#J$S8f5+(0jR0X5& zK2#IJ<5R@wvyRboS#D&NKma&_+HQUjOV|@!tt+I!E>_>L;>U2mklkQ+jN|3MvDUqR z)Ol{MFzacLSNd#hTT=1#nTVj`!8YDDfQGrLCiQB}E&sir>`x9NXqC52*e$Z_mRS7I zhA*iEDcaH7DhCn7`eDPjbl(v5_+k+}yq_*nhlhsu9Y-JXCG!_Oio9J7tPkMIEzCgO!?kb1x@}eG!xy8dO!j9L1@_@$Iuqfs zfn5Z?ZA=6Dp^PQb+*_s<2YVfd!S!W=xBPon%|qcs85y?Chg4ODyh6Us>o5lH#-AeA z3xp4lq;mKVuf7jPF0+DXS&6gU-N!Ky=Iu)Yyo3pE{u_4bEBVHIb;0ns(71XD%uQ6= zWZX~qx~n7g*mK1z8Azg4p~eO=@8G`X#OsR@QEB*JBgG3M#(s>5TRcpo-R!O}`6j zv`U#_-DBS`QjTxYBRO~%e4_CA@*zx$gF0B4Wa)o{{ZF?#NKe;x04?Ve>oqP#cAY=sn0m$mMKlI2r93#6~s9vIM zHpkhO7U}1=G_w~cKCN)GKo9&h@$(G!FvGQ*-=}JAOfVBBY(&L4BF9hFvWZS&?EHh7 z9T-d6$FItKVAjd~2p{8{R&$-(6g%4J+_hzC5A{>?GJ`u-l^|kQC$0`AvB(KwV}B>RV+ToSdR4P z0;-H7qw#Y;W*lq#<04LK+lLh7XUy-z>{(f^ynIaV{EdbqyfCwU#8|N>NirsJlm_~u zlVtt57KG}Cq{7>rZkRqcVXnDk?Z7 z1lU*HQ^0uXN&bq1`l80-Sg}O{WLptOfv+1U#zpfoY#kwKtF5!6uj{-bc`hEQPw(GK zN_E32jTPT{n%q7SyEwwfVc6rGn+ylNL8=VFZ&HtwpnG)FHG-iVpYz~vdFrZ)?_qqx zx7IM6ZAErMk=q4Ltx#LD3pGoKrvDtGco+J1)(eJiQ|)&09-z3wbTTJ-N1z-MBcIR5 z0l?8>^8jb@1Z5j>9j3R)HMvP|?xi>)_O(S}TQX#2?qS-fHPljI^{@ciXEI&kjPlEa` z2hFyNqDde$+dF?Czn-^9`IJgybhdt@uYRGfK2pT=1xl=a4$ z6kNB9Ujo#VJvV@MjeXyjIr41V=e|*bBO|%vB5Dn5Z@I%R@}le!aJ^qrM>MTc#haw$ zs(qbKe#pO6?|xG|C!?iMJI7S+prm7__#prl@{8MpGy?>~WJwmHQrpC)Hi=EW{aed0 z@oyKq2*uVN(_4%qQjN*4{tQ+qjrAUUMx$46Nb@p9`bk)73QH!qU!cJM-bhrc2cAs5 z$2}1XLH9i?e_2qwEuuwiq;buxAf$3#8A#YJ0Kzr(^HR1^eKwYJAVYBl!UN_4aY#}G z;u400h)s*4pDu)(LC!5Oa&f_Tga!=&1ETiKrY8XkNb>suDVbsWZ1$71s8v##4Pm@3 zB}{7y)QM|1qnO=}&&BV#;@(4&CmzYZ7}fsNWpDEPhisOo+_Ec0Rd?laY|4}GteB5* z$2}cfcICNw`2JQdOWvm;Pu@qn1a-T*12tknvkcSu6rF6$w2_ZQmZzB>gBKBQfl;(b zed(6&0*WNQ4YrAo1VE#J&;aX!Z;1FIrK24kJU&}r($-9-7m;(bjZ%516=@+1+(i|> z#6i|j7*?J(mS9v-a1Ck9J(^5wT_K59c@Q)klm`<0o}jlW0?MAm%Q3WoQq8OJ^i85G#sR+G9 zwVYui!5QtaaVt|A-JzpN_CZkuSVh~>sz|yDU%Q&A<>By0Ifa(Y;F5TLjR4C;u(}0V zDSu{g$Tsbe<{d%pG8dx#G#^))>z}+rr)=_{0XHYlc;RZCnw073Y1r%}663!dv za+567s`|h!-~rY^TA(vHqJz2Y@$mwjRedVnvt*kPO?~}$oeL1?BKktkilPu1pfh+A zWjQ#ffrmR%?2v6OZ>}9W46K1OjHU%G;};)lAt1g&d1sM;L`Q%-`qZ7Zj|{(A2jcos zvD)Q6Rkz%}rtV1HyYR89K2*P<&QlB?N?=KQyoh+O0xNW1bItzBIXn(Jic{o|-cVF0 zt7cQ=Dk4mQjr^bGmYy`5`ASKVE-ehRD|s3mts?s;S2N+64ANj`t*oR~Zw5oq`W4%0 z#Mos?fg+MXPll2|$Tz|gidQaRL4k8CAryJoU=SYU;s(2JBH(bft2Q^wW`n8j}&k1VicVcM6P28f2Hy|2I+e^ij_vkpyZeL z{oO?0=wILbo`rYz+eL?RpQP`EF@(mFgK{6Dez$yY9%ro@>$S_H%tWTmM#cpM+SWGg0)|wtm^M5o%W=T1 z*WM$`ZiOk+8>1IqvMU^lA6oeeJqqqXQkP;LL|H)zR2i}mK@vZ{J?(Wg{&n75NP!1I z(RY{IrBv_EN)B))W)iUU?VYPYwYEQYE2Ea2?Vo5TnmnAh^-Aycc+3_YdoaAR0*xU` zy)i66non!;FOfIM6BGm5DaGJV2^60K+94PCjZU*1Cp_w98h?ETEt!88uToeN0@V${ zgDTGf7gqcTCs6!ILJB%934SdLYp6bz*x(>~3K5=$XhZw~xK!{IA|!GLT(YnLE{%); zmnd`Qp&RZ;pJ#tTf!Z1Q!VosW5pV;xLV^Rf#1M0zP>cXqS?nE_kCT}R=~fm=DZ z`Fwz@fGty#hnbGbZFGb}B>yhvS~Y}=!w@5-$8D7fq=%V@b;M(H6p?cOAKWWG%RBN` z|I*YU>@mk^+H#W|+GM?~u`>L7P2!>amSVY8$VN?}hTeJuG3$4HR@!=?+Fl}0_(4`_ z2@p$mbZ=8cufb`9MtU-SB{NPMph-kfNkrf#yhUKUJV@h9qcbEE#%b_1?B-ZiD{&@! z&=2?_Yw!!gdpQ30njdSzy3kF0p;aHs$%DQSa-*T_*Ew%LINJ#tG@`Rd7MA3SOQ{6mfG<$J z2s^WnfwI^}v8;;``o96UIcvIO>yG_n8TK;JhMVO2iwB>y@G29$huyFpwHNMu_L|nL zO5W5zTg-m{gxfqW!@QUhd#Xf)NestAqd=KEDrE9>pAKEN7xeL|C1Gl(s2ZK~!kWoM zQwcgGTgn8Zl>Yfh<~kE_Y4Q`vg)$$~g)*0yntZ#$u&RQ0v}?0wgZS#Zk-MD3^H-7y zr_U1~FHtC>Im3>%l&Ar|XxwuxDeYN(&ue$3FpC{6jz3sG-}s3_m+~8(T^4LSlkRB+ zBi9(td0SEUV_-AmgR)x5Iq`NIDaJv&H}&&k#pnn5{_rzJeN5MNay9_4EZF3nF}90E z$Z@H)GO!eM0yvg~B^P|6H|{?2?8oH)tRJ6&u-xEKAB*U+KN!?J5wC`PVH-PifF%HNsB2H&j+Z+6MONbBrN#7L~P5wUQ@ z`T~;tRiEuSm1Iq!9DS$>%aDMm};osB}C!k za#t%+-z~uz-One&d3w03|9;?y()1W_d%kZHbiEdWk>dF=xA-_{8?XY6LKseNU(6y3 zJt?A2ntihc6O=>cNnD;ZIelMct=S%JoF)4*^NyUPRqDv@!jb3Nlu3b&{wR*nthN3D zXbrb7Jr7oOp~<;1m@#guf?}+&Vz8n5D;#dFT!&SbXF2N1xx6*_7D+Rj*kXGuLOCO5 z@e|l&X(Ql9pduREgfwalf#l|AS9Nq1So`bCf0_wHSr7kiCX9);d;L!{L0C%WB)sxq ztnJ!XebUdBxZvH=8P&En*~Hx{E4t3Uw!);KJN9@sFvPw(_FX+cTZH~z1naL*6|_}e zOWTLiP1|$=3qg+So5sbx#9G?UUBc4h!&YmJ9vlSCdHsw}Lu?*>|C?V7#mC!5o(am5u8v^|734na9$@;Ox1)TX`o zKlRnrFu&F>-FVAi^EY{~vY&Xf?*5x8uAJID$GE)kQBW4Y&nL+qR%XVaFu#&Xo$D zB8q45oIbpr4jt$FFY!`-OTAfK_$ikvUpE1gi_g|gmO8QFL1Sn}4ivpk5@Rc+ZdB(> z-uC(gQ`SPcI(Fi4E%@PHoEO;FDdFY{R7_^x&jh)$us;h#7P7i=#Hj}OF}4G>SAqB= zxl5A=2@3_~d8@@I+DZJ4#e+<^YiJ!4*37B#DG_Chk$NWGoqha9S}K*Bc)p9mF`BK^ zU2r4SZTd1MuYlAV#m0jCL+$e&WOk~8L2HP0Gtq_jP>TB-x6BIaM;KuxZI|fUq5|Xf zN!#5zv@Bo2X5APagel6W KxU^Hk0eid+)%ky+0CP zS#C37!&4r=Ngssc#cThInA5Hqm%t_$$w-zm4--f}OC5#R%X+N&Pm5q;I}di+buT>$ zBupm@smkLgui0@&c9~o}LRInioeT-L)X-o~>oq%4#jBwg z-F<75gH65_fogPg$kxd;$f@42)czN65Lya41^5V~}^ZbHh7gIZE4+|=V9E=dA z2nl2Q4|<61j&-NnhkY$N8d7^9#^7Ai!rXnIWQh09CKTOqkX={?JXu-w7(JdQ%DQm2AgoQ(cgq|Cp=2rua&wV>Nw{f1$m8EmsjO zwv4%oVpu0iJ@X)d^5holqXegEGR?>LhcnJsA`=5Wc_rm`r=z^+B$1^8Wuc#0!{T~9 zfw6q+g4-qO(aYD6mmC-q3cBCTD)h+4;7b##EukiNrS&?hVJj^KeaZhjAy>-I96nsj zwig!#dLnWv>?i5?>WZ)I#zZpqeLa>!g!f2sEY(&oVgiy`v2PdVDaJC;Dk27q*easQ z?Z>ZG#FQ}eP5dx(L#;rXJ;|z0FvDQkWGj)HX_};XzT99sc>*GBAX<_Wyd|1GaHOX0 ztx{KRIkYA{4hjXrW&+5H|KL589YY52dtP|mAE*8~N#CHyBlpz->z?8UBhq-GVCjfl zp^eyP9QkZKyxVcm>mJ;(mxm_OMB>C7Ed|bgyYamKQ zX6(xJD0N|;i8v_tzDWws8#Br7%6#uLa!>&|4uLchfeEc^Brwb7IU)}3b+udDTk@6U zh5_fVP>rHwy;@fwoj3Y1I-sVPaEA8VR;kCwJ9Pyvh_?@WfoE~qz|epzQj0cSB}?zp z80gJV(xC(j2;JLg=IqFU60C~6cT}8Q7yyKR1FeD0P~=k1Qj8EGTc84de^;aUt|k{B zPY}ytD9vxCpkssZbzL`9DiphTXy6bDW)&*Ax)Vk+I76CYd$Pp@M$B8|$0hb>3H2rj z!AQ^r=^tcNMa7K-LLc1do*N=*wH=IML*z9qKK4WTOZ3DpXx!F<2yL?M}FZyhuinS^RK&fX9AphHMoNh8@+ZE*dp zp0~j$MR(J;oN4RPr~Rw1xwTbQc5!*L;ASm<@%xCqsf#~*O>eTy_pP(Ixo22qy@tC< zeOd+bjnQ9p<_vK=FGu7saEAq$xYSS#5iB(s>8hOsLmZ91AQHU=zP%?zxTExq7jl#E z_!IC1+1?WvX?6C=fFSgwei=a4o7=G$>=J1gfxGarI~=#nl2aIPj`mVrXRjwIH0+!$ z-fbOFpyV3=)`t*a3+6zoPRfQ=OZ`f1KUqaI)6M+t4a*== zHmgYnkxHt_3q`YI1&J-aqTU1=%rCTpN`+5!Zh7PBrWIzt^>Z$-^zx|MtE08+m3=bep<*+Qz#X(1#G@H{! zs`j3agDt|c194y$lys{&2}dfe5E2=En*jm0M<>TEG*mQ?xNpbU;E3Bs~inP@m+nESLTd3+x^j9y9e#jRPyl_xLX# z(9-OELbxij!i?X+S@s7ll=8(_6nYSAnIzf+K5WZyfjY;D6yb%;W!090$X( z!d?#&FaKwSQzOu^^DLZzo(cpLbwA7jq-n;dMQGM7kzwC?Kgnr}m`3T1l;yG_2>2v! zam%3aakzcy3@&_22D43$C~i!DDwco{Hz6tc^$wg*;k9*Tu6WQktywJnLfLvQ5{T?Z z3)#Bqkj1rN)8G8NhXb;?1Hp`($YSq#|zS_#FdGx49=pr7GBI*C1p;;lZ6;}_(%hbmH zX*OJOU&tT|(&9{&2;?|=Zu@`Ny?E|0#TV z|4aDFe+yrZ=OJwZd{Uc};}tjO4)??%&sT1w+uqFH}3dRAuH~{VWQi`xrd`RZo&9C>@E>@ z>k^y`bBsaz=KrmEO&mRgpR-Ok_C|&LEo(BMeP`-wxfoAc7mSPxeRkQquai{uL+Ix4 z-a`BOK)gHylLAK3@411f{x8iN`mK3&3Qqn<^G0TnsG<0ZFU&;@DKb_jlR{tp%E_IT z?5R{FXT(@SN~%x`jl&|X4%rMl1?R?AUW4I5a0@gx;;FQ|NkqNJ{@7iQp6JL9hwBrp z+ZXMERVdAej;27IG>a+SEnu5K!GsCpp=#!WW@f#eymV(YSf7rmb$7O;WDgtCQm@k} zK~-=rLp|x|?jT0|9V;}$X?Uc>qIDcwPmMyOg`TfMalfxbfnf-O&l7f(utDh%oyO}| z*fj18NbkdpwB9}tyFFm>K=T{xJPVdD-far@T}0jtY7&k_gtt~fk(_SU@B3BxpX&F7 zlMb2I>RbKdqNBUB^bcq0eW5*g?|v;D=1yP=&@BG90?u6#8*=h}Kx>-jEupedvqhLv zh@_um0sT_m@-E_IDRI%g)Kub_a|X>r&!vMt0DDX9#37O}+k5j!S+`10*e?7bt0RuH z`c&?LMfQog9oC^eseD!EEwYiU?Ctb-r7VrO*`!SA3R2nb7}HLNo8;e7$huD(+?W1Q_>rA2uRR(tK?tHqy!i-vcBiAmXR2I6i zz)FnX^6#18uf>P5xR2G^yK39y9gq;)i{Oh8V~aTM;B_bPM><8~T3ZJ3YbgkPdqu%sE0l#I8O`@wG z^;pQj8Z_28m70u;y$@YdI^e}&tgXhMLJT#3k^g}X{*CAcaO;j~mwBpb1xss|J{(aruPsV!LyJUkwr@%vjp;jm(b(#_N zdb>-x6{%lk{(C%HKE2*>9nr_I>I)?sJY1FzkKgT|brCK$j?oYu3FVTE_RyW3oW!5G(wQJ+SE#G}x0*2qYlqeqJlz@M(-! znHF_q8n?~!I^rY6U34`UH73Kb(rUEqiOeb@VjLZHZWr45dx36#IGQR7coBfS4l~sE zvetadYE6+a#K81;R&X?4YgW#adwGCn*e&TYzds*9l{ob@sv?$O!5QAMwb<{cP$alw z9j-1g_ZD-(wtWwe*Q?R%#v6H2u7B8UKeH#bvt|eXOJ_>h;0aAx0Jt}v8w<1-D1exz z38XiEcQptCi@c#7*oV1@pn;NsBh3GK#b|+O*vgrSODqDBT7rn@!7kK&1uM1-QU>?b zFAasWO^IH%Dhra7q(z?13ou-#>K_TGJUQJKn|Vv9w!> zykr+%gvh?mM(vIAxW;9s8=+eckF>&^3f#8PAw)<>2d^tu0OQe;%hB5rb--yjz9E7I zsi7_or@^Ux3{7eZ$pUkI1K#;Y=79-k1-^g_)Oq8(!*Ek7|GzYBg~mJwEWTeXPvq8J z?2-u5Tn+f|6%>a&7mP<-l7NnVk^=Zs6jjF^V=c^}zgv-b1c{S(kWOK@ z%L();Kw;vI-%D*M@$Pf6Pg%R-?{CfN|ntCS{@1Xr-y$~%xAE*b&4>>`7n34BtyBSLt74`yw`0|z` zZ73}RABc?*%5m8^o*EA;fJFw7etHMO1>rE!$+v^m%lyV#{5>XyC+pS88o(=72;>eU z4ZN}H-rA5j5q4H>9`&J$|F#&7l8Q4sFK$j~(uDelg;uxxjWO%yK|pK>g2ctMx1XgzLt`+2u;l=F^F-6t#7YID;b6N-Fk z4l?>mwz-m8bcikY_(@yPaJR7cd(H0pd5J6|<7jjWB~O<2tUbj-luOoEJg;*!!!rV9 zkvcr-^>N8^QMp=LL?DaGDenQQGHYnssC)FeI6($ktpmp;c@p7j;nr)f3}B{C$zZS9 zPq-WZcI^b*&KKAcp!Q)i1$RD5@P|B5mTfB{G z-48y}3|9`8^cP78hxz_r5u;mpHA>@i+)AG_F=D)1EI~uUC~nZ?8nGcgB=yuaDcbKd z?eA{@3nH>kI^qjtM?Wwc)1v3-GPq2IE(s~k6Z+I%7@)LW!^)#rbvfc8At(rj57o=s zhSi-m_mPeN`!(xSy6M&`s~7SVK3;8XpyY-y22qlo-+N8D@S5DJN3}zHG^y7>yC`!0 z+e(OZQGTmSiFt&9aYCOd9EP;oc6%;tqG*Z$;Tvrdp|9kvH@LB6q5P9$FjC@)1^Y_omvKnz)s_9CBvhR9K!7!!t3nTZ!4@bV16 zU=Oiz|J2NOqS+HZwDvjLQeVqI2cRzP1+-@@rkN5DBIAE|kj3OaoAjLb_L!R;<_mac zYu6X*IPq6k7Q2d1aHGX{%DI`(kvTF3`$>b2o1!3PI2e2a%~&69kdX&SEjZ4={Iz&8 z(0%Ee*antItqVkbSM+x@6lSj;OQGtD zG30(aaa@CaLIS}a#pW86L#Be z@mw9#X(oqbd6qgZu8p6)o>`Y%`_xr5&a45Pm>Odxke~b`XnGPgatZrMvZLT+zo9QJ z8p0QTUk~@m(R<;{<>4PXLelTL=KJa8cK^H*&T+hu!??3r^MAxH$tY24(Cc)p{d_pr zCfykQwrlMpyk$mp|1o=xxaH;L=3N9|HRa{$Uyr;OpQ}-BSDoHlS1vr8pRi5ySIr%a z@$E6d*ZiTIn5Sd5rh;|~*#;-g-8`?M8iPa8$5dP&GpVcCpDaFtrV}>|+Vq&)?377z z9|>2;>J9YbbW0014eY5mxXq}lmK{~N{-QW{gjxlkrGi+Nr=`nR6_^E4pPE;h(Ni~K z_H119O^>TD;f-(mZsUU(_^5H@nJbc9H<8h0VlOYfZCA%*;wh7g$GKspiwrxBgsVwj7P9bxe-<9FoS>IxS2*C_Vzp3B6weNCsXiC3SsS*C|36^~J$n`P zl3i@KRi1dGYxh5_q>a3P?4R)EqvwBwFa2LgQIOet@}PZvB>!4PUdY{W$bF|T+ZC4> zQTG1ZF22uj|6g{IlWj4y9IBg2;p<7rc?ORS!sv8P7-s8<^MME|@mH3wydwa zf$TL!ELphVXE!u&Bu)&83C1O-sd{rbs2XT(q>nSItqqE}Nw zQ?A$v6UcDiqVnxHH<&t0IbkQ=mm`Q-*;De?Rn@zWOSbC(;o%Fz=9_Lk+qn0yS$F>r zYwrMD*|)C!#Q_?|k0p`3=>v;Z4|M$%<#XF;3F4AXSL~^>L<#Y>`4^P^G-#%yJ!GK|BY zNnOProcK;E&Xrq>CPP%MOvTdW)cQpj3|~<)l4-6Fl(hnB+QR$ahss-IE%Yi4(i*BU zt-U15&`?SoYxRu!Q&gv-UdzGK(!2wmXJ9u5)mUz5<>5^h+7$)&ru z2dv`1oejLwoWk#%8cScI=!(8{-Y{|dOG^5gW1gq!Nz-E7L?|m~oc4(O*X#~|w_KZ> zz^pSyu|@B+A9JVZQS&C#S5@1jfmQf1*K$W`uBBBOrc^(~!!W3F@Rsd+9$k(In= zLXQF)u{oZsol+#jyJJ{M*ab)r>Ho1$;HUad`Gj?FvmXE^;0ecacbdMA}NPEJfa zF|nme^EP^0Md7yGky{yo>2%K?t|JNs)!IM+t_OG0WcP;oS)lZgE2inMu@YNjObgs7)=Z3 zLSV;FSX`B1rp;HMR*U8+uuEBQ>MUd^uX3;C8^pd+!)?_NO30o!{a=pvAZZ z?ZZ57bN7ikj&aAzespFdy|}`Qjf1M${6nGkr_e=;V~HtKFt`?xlq}n8o8bvnMuJ&O zzkgYMb@~Mv8Za0i)#SQzzGs>CK-!%s%@MTv1ZZOQp#;@pa}_ZWJY~ls^rU-PO^nWHK*condXN&#~-> zT;80>QO5i3BJ+zwux~2)a-`uADkHf?GJk$YjfQs+ykU`@F|cm5YN~C%eEm8Oke8n$ zcP)0JPh83vIumRG!oozcRy|0zCS*xdF7a+wOFEk+;D;fylDdwihGvvTF<6rHB>JT| zfufsA+ES4^S1e5&a|ecjJ`?9P5UoZE@AV62T%B;jb?&-g5@VWoFi7T4qW-KF(+{EM zSco)cY}E-v#q9RNo_rDBSj^?{Mp~tzc*K$+M@_5QWDD<#$c^9XE1+CI&V5lv;> zp*Nv)K^t&Fns=VS8ViV~=(kqz)Zv4YiN}w$1^$&_Go|2BVS!cfK62UjF{dISz;rY! zx}k^I|29mJoaxv~%Zrp#)pc}r7MC8DLvpiwv;(ZSdAF99@_=VKG;sdZE8;d=&-zqQ zX%%hBO+8%b>_bS1r(yfQ<1n2P*>c`Py=CVS$>F(Ff-iCMccZy3uOjl|<*3r>}hPTq_Ic;k=Mua$Y4CW>bc{J>bWbK{`~ z#(%uavZpdQ^S_Lruqi^|B$645$`~YKhpF@Stcs&rk>%alqPqRQbw*cZo1bV}5Is3y z-aI~Sy*J%7p62rEp)p>o%M^AeL@Gm%m+MvzACKDoiDf}0i!4SJIx8;vNdI=@0OJgP zpOJ{~nNd)7_qDC-2|w)~g->QcXC;#T)w*@|K-*r7oJ5POGZo8G|UD@+m?Mw~tjiALF+S=?yh z`Mfs#(?Qs2N1jmoi;qC}Vd>90VLLM2Q4IAy2ES2%W};7zG`znsJzUqzUEbL8MO+Y_ zlI@J{c0zcG)Rk=Qyy@}Ay`H^E6ylj@)|_f|hG$a>eN@vey|2ZHh!=1XrZ@4VZz9?D zC_1|QuvWZEcA@VPvw&GvfHO6sg1i7x`>hX=nEapBBTjU_FcLY-?&oohs$Yw!}sfqIvo-JFzSGb_U$ub4GQRnL!@ z9eNoZIt=c**T!@m7QKtQ4{*s_H4IJ_>7B~3c@>KVSsU!H z#`?LCSuE~GH8xpaZbc_<*DeeGg_}ey+y)W-U&aBl*Zx@(5Fb)e6@Y7H*lm>`i+ih^ zZq7Qzn7e>hlX;h#&X~f;<|9>yN4^6&7jq?~W!U=@ICAsE0m;phZ8=o2rP=fPXzOZ0 zB|4X+Gg%%p zfA(-sR`Q^r@9Nt=MI;}ci;d^ZQ9NO>>TxZ_?R<=UTW)v79Cke^t)qswR$J`Qbr5r_ zU?tCPSJ6@*5QLT|wQ*ty08f0gLIm|Jh$K!3b<;zr>nu&${L7ZQ?e&dt_(D)wOKk+< z97@#oClhu~YXjXnm*r)9V%Jual3aM_Q<01nG-Jbd4Cc3wD zZaw9meqd;f-~gX1G{=|P2D+1*4?0z=V$1T?^3VnyRI`QvJ=xgPlD6_dxzmXG+%QIX zuVGS|^ORa%&{jB#hCng0nT`lbw!<5Q`9nHT@)|bwCv;NiZweq(AaZvQSyQ%yV%}di zz^K^zK&-%)#x|D1F7%MErAgIqIlrwTU*FwUlpK1!Jv{I~B^PZmr4R8xQ~u^RNx5=q zVCKGW#=I?446o*inRN>d?6eT;S>SaH=Qf4@>LLDxjM&kU6Vfx$0D~b74)$Y4>55>x zBD$qO)%?7wq+_FvF*9A|@L~q1$VeR?c8<}Dv@oVVjt?ke-*(K<62!LcWrN5>M2U1@ zVhzdI`9uJSeUr>~Z5xSQw@4qG-4>L+hE$QIHe9fqgsFcPT=@EAlNT1RBSYohw~uxa zXc8p7c15AD>3$*53=w})-96j(8HUFIc}iIK^8*s-hYj$LceJ(xu9l;uqU!ax38wQd zl*$;rwO=sEyhf-SC^B9)lcT>cBM$UfDR?6#7`u>$L3=y~^zDU7!EZVr90=2*p*&=% z(K~i(Y17^NN0`5WhZer0$at)jCw2<;w?yPYUgUe^qB$tvGa2g&DDX@KeE%ap2^QmrBom=dAgJ{Awuzwz`T=~)* z{6aWf&-?;vd(ztdr55~?U9BE&72i8`f4bM<*_3_=gYu2J~>C@&}U+9<#*Oi55{T?X`vR?;CR zheG(DTrcMjOn5QLI~!h{b%PV$g_%!gqUU1b|6N)9mb%T)d@C)Urf7wWEv=reZ=#dY z93T;=e2#YSN1Ft@*PNmOL}6^PocSxmB}IUw=q8A@GJlLSZpJ70QP9Xt$t%$cnhu9a z=Tfb`p^kjeS9xOBm4cg9|2~8%=t6FZo%yg&*SQV7&wWeAb#hcLmOWWzNx)m{Uf6zx(KTy01AFNb0n=eE( zd1awNrOhS&5aW0+wxj>kS@7z?^IlwSPQ@&&N^yVFQkNPF5PTw30Ox$`aMX?5HqThP z%~)4%jkTP3+spH9fcT?qNJ8fRi@yDB@8AQo;DRP~Bd1`=pr`wLCF-5Ke>Q&D;>*jL z;LyUnt^0%V2|lLIuhcg9#TLpRdH_4@yp&v*I;%DvA9lHT=lXTCJ$+ZW%)s=Fjb-=^ z(>^;OX$$q=leP+Kv)o&lSKD6CT{~y49iN!iCpC*I283?7pu2#68N*WA#)SW7J}U9hf3>)i~_3TMP_zmdu=B<@D z0phq<#9F0*C4yL1x222c<(PS~uNn_|@spQg4xHRG?e|Mx(O;iVoo2={@lm5F?zW_t zo@sbB_$NIjG`L#qB7cSH;Q#;io@T-S(tAc|H;MEiOjmVhQK1k25qzlrN5Q9S{Kz^Q z-I&rkCLX!P)z!AMJud05ZH4dI-?kH{ojBe|V2WcJ)%6njKE2JsZn2+UXhJ|A99LAu zt|EFj=I$YYfE99D$b_(*eye#$tUGTyzConKIpeg&&sBu(6D5rL^WOp7P!9Rh;<&;>+?u$+Hmz9lJrt^0W<}VTO%}MfFO(Efa`Ij$NN|Uu@-u53~EZ&qPG;XNKH#D8x z&`h4;h>IKjzO@2eBaf3}ZENNGj!-ZihOYdJ0OiQ?HOF#U+JUxfMW(_pJRO)c z$HKdnzfNutwm+MZOgEa+#q`P=V7+S|M{iVhVT^E>o?+R`g)snc0IjVEpb*nUA{y?; z7$5Ixa!*Odo;+0|GMll^tSTJ~&L&D(QvpvkzSoIwz-iK>BxP1KmpZ0yp9@6G&2Qo! zgd=7SPQ8$Ua5klcjg8{st~2qu(!<0`5+jSAO5&2P^kSCKv#gs^!N_88Nwx2LAP`{_ z4pX2}r8uD^TbaYND{^zaO{$F}o~#U{Z7G>plWZU|EBKO(KB=kihYjU>pm4IoR++e} z(AqbSsf=uz5YEt6VfjhCajslK#*%a_MVGWrBd3g(pH`BRt;w<^2C;fcvLoxM)~LtC zJ0yZ$)X~R(_vSmIBI$~_n6u1SC1@K{k1b*8zH?2SOYCemX?GD6gSYd% zkBw;9rcCprKNT@UiVG#JIV$q&5_vUBsR`=YnxZ4eOxJ--5R2G8CB<@ivx90UapI`7 zw}l^ew*AfsmI5+uR-Kr0WC=IF=t)T*`j5@i9=*`B_i97 zNtSLO!6eN(cW|ro;R9sNEm)iP0H{7}uvxI?v73k;PG?Y84(FB85#8Z8ja8`9>o&y^ ztNr;hg_z5S!bD{ox}e8<_;s?4;m)sesVwYV&cM67rNBE;#KJCRV?nPnz|YjGaX?2! zU`KAiN5B)4s=$3L zW1}bH5^@OI3rh>yV|U~=O^1!yc;qq;{sEfeAk&m8ti5X z^Z|NiRs-4uZNp5$qutyn<{%I3e&NDvfaK2tdIUNLUP{4OZwB^ruE_AAS$jgy-~_{T z7@xn{VcOq61p%enn|;b7R689Q)tAWeQ8Jz}z5hx#>29M7Uo7aV8i2Tko-q~!AHJ|&jxr;Fxl(i0p9PR|oL0{8Ug zf}F@cK=eaY0BNXPPsiV4!2&uaa$_P9Yw&sqJeKrb+;xYp$?nzq=uJrY%9~E+zYqA- za2zx-GTTwPC=d5U-?m#*=YUVk(8bk}l*0MF%mO;~%7euc8ki=AXn3S-+&2L@vT3XF zX~We4Mg@?Gv~{KmLeALYJwUN=SX?ULn#sTT1|`*U11&BIk|M!DCST5tn`d2E|{p1X-CQ(1?64NAzxM6oV2ErD(K0;W~} z85Uhbv~Vg0K?NBj@HiKHr2`#H3BU!hC-y@v)P`gUSoLQU9xAt z1AweCY;Qs@+$9}_`i1#{b`fEGg%psL>i_7>xfE#xx51Vn<+-(OV4$H*jlN8Ds}k0} ze`N7`9NDJk-T(O;h4yFd3 z1~)s&>wmCO!KM44Rb>AT<9Kh?&EF4wC0eQLi>4yof$rYIr{1uk*R4jNxn7yrx)y?z zX-?!Fmaje5sQrFSn$W(KU6Ydn6Oi_{Za>n?1MS^%L%TmAoH|d=+tw~1A4a?ZEjz>Q zqj9E1o!JiJCgyOR_MGh*&9>3H#?+HnIXjrF5m;q3K9n*YSvcXc zvNe>lUo_$JfC{j%+B1i0Gi$-3pg2%t6;d@dMToEbaA*WtbqC^W?(rB`hyK#zK93jy z0rKCrC^W-BJ%$}b;s#;OYbW0uQDV8$zw%;e*bos2=8GBysp=0m>EVw$=j--K!@tqU z^j;>b6<|n%f=Ly_O0!~=_;`sGwzkVEf4mV0(ve z_7NNWhIUS5KGDsSg)MlopeqnHdhj+P`W3u;L`HUFuxWK*cfwaeU+>$cnd^g!=zuN$ zgbYjfIzJOJrL`})>i2&BdB~zSfvVX)SyC*Lcbl99ZxTsM7Dz$=5wTvB?>a2_h5!wC zKd>YcO0Sr@9|^`+pxjwBSwAsKET!bKCX!dBeq_~PNurY3pe%?AWm8I`rq#MAEV(zR z!j2QksTySv$}QPT7>Hz3?goF8M|JbOC4wAc^GVd9KE#pyWMOd4JS%%LEEp?G1m2#) z;C}e2if)c~2a%S#{yC)z*?|dck)9e9tG;YviMsJ|3aq9CL7)mmXWGiW=D5)vhY;qd zgUuGY!LQU-J9UM;Z;!}%tu77h)QaRmLlM%pVJsZkNmm};rZfIyW)gi+4*64-RoDUA zJA{r;_s6c8qUax4-P0#Vh^M$8d>MJ% z&BuPwhjLWguP?gweEa;17~P;Sd-~Exh!FCKXw2ygAe^&l9sD%K8zHVF(1g2!j=8d> zHCiV>4lP9Sw0-unwID&1X=O5OY8b<3T%Up9QQsPSP~?H=YI*N!`OzPf#=&aX^7pX~ zZW4+NkxCwd0t_FmPoyRvt0w12g^(B4anM^U*d?R_b$NtaE6eW(i&gnZNM$X(O4bXL z1dmNv=uj#mNaf@dF@sB{u@FYqJm1RQ$w{J$m!vt1^b>J1U)Te$E2YATP}K%sYyvL< z*9t%1j~m>Mqey2^!YN>Mu&}_(7 zfJ!wtc?6@St^(13sqm?lr@FToc}#Xt%ud4l8L= z^64~q+Nq)d$&46I!=tUr7=H*n2VMrP!c+#SG64LKjH{F67ZgLk4?OA`gMD)j!Rz5} z(8GKHxA*WG$Qkmm2X;(vlM`qpOn4difD1?*C>AgoMi9eTNdzE9{v6t;K&*q4Ib@S= znmAxqjAsNv6J?%2aiF*e7#u(%esUA{=6%?QPty$%yVL+k5;8Hr5=4+Y@By$J1>amc z!QmRbKk&HKXSF@%)y94StpR4=5Q7i&clK+4XWsyrJ<#9T2jT$%uqauTAV8!*V}Q{> zhd(-hVIr9VMhBy5`T&bYLj%6?fsz0tiT{oiJ>I@GR>g$wewKnG0haU&2 zLAc{lh5>p7x&uCpp#(l;NakZ2TZLka=i3o2i9q8NAI07jcnYWQe=Fwj06U@yd2Vaz z?E(aH8C;!ZMCSZA7sl-_4hu%hE8}8w$`?O2EG||^r+fX}CA{w)3c7c{>_fgfPrmAN z;*NLtb3PRCK0K9$_>fv?lo1Mhi#{NC+F5hPY?m4IJ3=T5>&G&@w%?*}{6Bu*zML2; z{khu7RWYLlIgi@k4)P|h^aDMQ>KGG_0Q?1c~zwz;2uJRvkNUen;sw13lKs!>1yQCJ3g zOn7VOv-*DEus?4dDBsmp(}xt&q~h9l?xvBQsvibgEn)EO+}cy$iAceXy73;oy+AL- z)-rcJg#ttx?4{2N#yh-VFT~f@bhY7^KjllW1L|zbla0>3XtSz6b(kISZm2M1E3Ig$ z;AxH%HKuqpnS#rs<1;n+L&E-6EEqGZ4US-PWUvzMi1u|$_xe;v=4d~Q!h$p7@O0{r zqZkbsy)TLr-Yvd8eh4p)o&df;gPm|Myz8s2L~T-5ro&bpr7W-RaiZQA1sobTGB!gpvq&DjRc`PnJF$I3QM z_VUypYUlIMrS@$!oW*rJJ5S@hI_)aLVZ=*!SjX`Y=(D-aoc-!&Su!J=B$Ab;x|OF# z9d|RDF+>`O$Pn?HV(~SmJ!?|&yoYqoB1&&OAH%br(CSU0m%mYTxhx^poH5s%lADbv z)}8hGb^9_p$1>7f-+0C|B6{`gSzDYV04ujMj7p~mpL}b3WseOMT(HddY6OM(OGya=MD(MNpop~VINlEQE6 zhaLxRte+PInj#u7|1t;RsR7JE9tg~J-5ADwHd${46(Q=g|HV7#{P_>>fcVx}DhyFt zSPHxS#O5@_CMQ%bilMtD|E3w+naLuGMycG0)~H`TnO5-#h9#DNB3Ss~hbR?1ftoGF zF7!CE65%doz{`*y)Gx;gaf-xq5Th#3dPQA;IEg~U#5MfC@!^9nY}vK@cIEg9s{66t zABr4koaTjqnEz(L};rR!$U42@18oxcW*MHgD=wi1t-UeFjZZGPn%Q4Evl(+%E_&NP} zS8%5VTzk(R+qDyh^>|P{z zLDWmA_GzshH$Tbg$yy4nXT}COdI3eK;uw}eZro!_o%NMGuE&gb_x5RxPk84+4|gBt z8F6b=4JNg&Il#pfcdF~zIQ@S&yWyhLqsJNB_|LHYHM_9=j%~*nGND2Un@pc&t&+I-n1i(_!J|)C&M6x1 zKQM!J*syi$&n6qoW9UwB??yc(`GqD4nXNz~_jgFC|k)@sn7e4H;Kpu1pNO z0UU1%$i4sJcq7k-Ssl0*c?>oZ4E={Yc>Vw64&vVfxPz4gl+%{d^R^iGQ%X2{k+5O^ z);cxFPObo2=S2aoQo+}I1yWx4+w0`ljJcXp14q7e1!py$Kj41{zV( zWeF?JDPK1Qw>4|(dL?uD(A=OIW@?P~LvXK1fZ;HVXv_MgYF3--m*IUu`GncCYP$-( z&-aOL=gNF)#ECL)o}<^6U*9WBwRPzWNESwaYIv$XR_42UmL0?w8okpY?gdn$DW`7b@G! zXsyVwWS&0~=L5%L=HAiYpRmAl8!&!S*vbf*_bm@}mg6hnX}hc4j^vEsdB+?5hvrGA zGVyjP!peaKO}g{)NYtu((~-etaYSEYE|dlY=(u+Ftj!lX1P2!*V7TLX?P|!f#G1u;UQK2`ZRM17Ahejn6__c|}+; zwwSZWiMAQ2i;gm5B4(Qv%^|=_O7Y%(gN2)fiHDh zTWBf|G!tmwmXJ_SZEQoZFggjkiAP#nGtaj#U+o!1M%f11^^1KioNC@x`(=H=;^sWH zSj*}~#gGa9 z6aLI#VZPOkpzK9W-cBZ~Rl_{dZkTHC%&-HEb;O&YG!Wb70v>T@Km6KNtTeiE>(?g| zgqTbm%d<}d_sQBcQF*#IE5x=>l0U+)bKjRF=~ThB?1$9E$B*WVf{na@#}(_P(!7;) zu}g)`CE7g27(JqsD!w#`L1F2e5PuL-M}{b5#yUXXnu^}Hw~YM*_B#Z$ed!wBqNWq| ziokJnP&+rt)VomXx1C-hmAiHMnf8UZDzYWuAGQOW@X;{dMd9*;CObc!-%j1Un}MB- zGi&%+*DYB0EY{{t9m4F&T?JiaWDqQ{OV;l^dL zrCo43y`LW=v59uLWDQB8?P~P-q9JnK5&Jw zifCYUk9FHX?t{ByX+`wjYU6ZCRW}!a4LQNDdLE#?g-{hfVrBgW~sGZs%*C=bXa zdUYrVK`}y%T%**N%mRAi>_Gbaf`LW2i-4DFgT(EhIPS{cj>=x|%HFriUWj@7uxy;y zT>Jj&=5HG7Q5P2@X^;EaZ{MAI3>|Fn$wVG8q9_MTT4_bwFTyjvJ89P$efPuhE8a77 zs$$${(%7tSdt2Hjwx={(R8Fofuhlc48kZ-v#g}dpkkX0NAU@aW>R|xO0rSD_7w=qy zEeMyC&V(Yxl5UcI4pw`+b8H=HG=`R?n{r}z9^R;`quQp85XVXQK7Xq>DNLa+9ymix!Z`83duGzxIrO@!T{rKGn<$be=r9YitHy+%_ zm*8DyK))Nu=kaibyj#y8yZa&D&&xW4{fcVC)q5wOT_PWD1`^ain z@N3k1GeyIIc)|*Qi0zXpAw=moTEw;3xj4+_BBC(#y)Z(}L?oU^byuYu&Qf%Ip;Q+@~K&k9gA zzF=?*lxWn=SdpY#84$!nW*o|P=ekmyXSzzA$nppjRD4d_-*nYieMU9C{9Mb5VBz(e zh4|3N^7rs++9P-+L`EbDbaAhKYMbGkG|* zK-StbVc#lOP8nI^ni0&7ZoXAJ!o~meb}5frH>*$Xn|<-)Ae36F+C?51SwG8%h(os( ze%0iOpx@?xX)sir2du=;q;RQE(0 zmcBo67fEQf8vY%d zw_rlTBPYlu)Y`lte+iItsuHMkDqkQTY!omThBBx#tr8o^VI=WkW~&3(A=DUhVo54W z;sJ^tc;bPy_S2#SNK!@N6PPMD5d<%d z4@g2~2}mMl{24#KuB&$CabK(d`#kY)lK$-8pxNSyE(Kn_-A18?Enu*IdQ^Up*ONy< z!dKoY5J~(*Un$apQH#Jsg7Cv1kxK1mdKG+MPj)dTOOXhf6STkg>(J_d@ATe0F+ggV zO^D*uk}|@&s%A~k#aiPq?Yt2-x`MJ)C2=x{jteJI)5sJg1~ZQ_o8fuw=WW5jo(eg4 zHDm3*PsZru^T$m9f!byQx0p^|91fF*ZP97MQ5#Ke=ot>Y;D6gle_AT9htzCZ$Qnsv zE(pk;6`;dtDU15@m(~+8P@JA!>e2$WcQS^>@HJ+`Yz&NcoAO1qw9G`yc9?$U75mtJ z(w|{m)Pz}zXx6rOhf4w81C%DN#EGbKdKw-v*w{UKxV-Vmw2UZ`sf-yh=1u_1y%4LIK} ztsz2V9H&KVl|5qT0+;=*mu8!}L@+BEOQ6t~dE;_y5O1Js z%EPoYd2MCpogUdqFVyW;AhODdK<X?*p6tTf)ZV zZ*QoPF{Gz)kAlP>>Z4oPtLFx%a#vk&uC&fxgUR2`O?= zuTFPoM0_xZqrebjUW5WO+s7T^;Nx{fJMpDc_~nLCcUQJ_ot&kcT2>bCR37J?8;Ncw zXD&-Tp3-ywbv>GMUKBye&;8DC;F*7SgiQ~)We z(uh4jPa3XNW{gMxDJv+H*GrCG2I26KickQlveHOCKd**0G5veuhaf4$!rdVo6&rBj z0f)m*I`;+S{JO>bYma~NkCb6@r0i;9ipUBOq#Qqt90Ebx(m1oo=?xy+kR{U;g2<&0 zC}k2mI1~!ueNo9=tYzpZ)C>B`L{6q3G7Z(#YLJ)J3ge-9zQaleSU(1QiMM zK`&D)j4FdNK0f$AiVnL>)?DExgj@w(t#g}8X<7l;-=-DeTHjiz zM3BsLBzo={b%y9nb+p@YzY%V-p9DE>IEz`+HzL{Rte1Uh8jCUYYp|HFDf>p4t!>9Q&K~j8jhomVkyLC%edougwgVZ zM0wbZdhg4JzGoI%XoXND0FF&$O=S(GL_;2d#HM>EPLK3N7?fUbV7QPfxjd~yS#c-9{ZlOa2V=j*A;jB9sV-K(`-+!LpwP~g8 zUPRdh-s#EBJai!8J65BhH);C5EwPxb&3t1Kv|RsT2KBi>e^RH$)CCE|ozM{vq`l24 z*sfZ1r%2Di0xifD=nQmZnhJEZV2U2@tQ^7J=Nx#H4>VbuRtD0i>wW_?CDsg@3Ch@7 zZ)ksH*DWt{|C2X2@S4STMrH-F0o#zJ&zy>^@0e99%;*QNIFuvH?(ug{P(1;)Z-pn9 z+rZxO)&U?7wxHe+54zwFv<`TTdR2;Eva6u7keT+CFnz9pXCTRZeXiIZEED2)2q zH3L}ngsTjJyg*(Xd@SI(6ck{&6h1&F=t!V5G^KSUGdf;67e5y9@(?X59tT&WY`$>3 zqB159es}3{?mG7E1rC;_1`aNo@{H_J7riwe;8^-H05NgvgB+n+3_-0BM;!f*=Ra26DG(t z;wB%+mdFbCwkyaQVFWt^-9eMH3%8U7Inrv{AOx8Z zuUG771HDqr5OD8%a+oIbX7oFC2@MEdK(6(O!8U1p2&-LynF_as&$)}Y(^?<7Ljor0 z!oP!kKG5KGPSR9X1{KZqI08R`H!#qu-2%M}rW^2$dmA z(ESQNyq$k*iQgo>FKR3~?X0)Ba(wCT$+D}|v;YgF-tH+8{E>pS^FLGqK03HA@|o@O zokSy9N^&IT?DE36>-ypaWc)I8W_F6*=sbz9$?rUDx`(j8VJ9$)D|+H&6{Qf{BK4o; z(8*l=ME}TZUh1EFekLP3Yt&&cyOnBrm{WlXPUq}ouHn>-m-YIH_mhVkg(qpW}c!EKmF+cz6tRDE>>c zOXqR=)7|h|KsAfXjnoZ`*hNnu$;*nT6Df0`q_Kika*AJO%7c(8xx6Vo(=h-dI7QB% zF!-JiD!#N*(otW>s~wT}Z7@A)9~A9DlsLF8%vOfxBm^lYdW9QT*H!W>_F-I*kKn)` zw{>>*&7yk}I4d`yvm2n!FB%RBoVeh$lzrY~{1EqjdM0*4c=_ro5XNkn1D5TI>lvtr0k5T~AhWII}+A5B_DS-%3IPF$0{10w;5^sVmGQV14Hx%Sy9%mLGGXB$b2 zwKv6Y-`V_ch1#7)MJYZ)oTK3eu+{RC;$AO@AN5A>zJkEWK%Gf|JELKacR zm^?qbpF+~cmmx?{Q^k)1;qFd-9!x1)AfX?~N?l?$9RF!Fr!ol`*1Qi)HD7O->iz_R zq|GjiM@8W2xSv2w5o{+CaBAd>zBFBPEvtveM_ubk?eBN@k3#B&pe``|H(7pOa2uJosk=Oh}wB%d!FxOG)CiR{K@ zfu@ijEG)1hP9|F?rU!A*Jqg|^cCOS?A!^f z(<9DsN$)~2JBGwFw@j84OtYC{^3K8J!l@&ZHj1q@UdLBBq(#QY#XDliXz+1L)O~R4 z(#vbjnpi=HAyK*GvPi+I-H!sUMUw8Z8M!pk`{K*2wPpK(pOCsA`}ktC<++y_bnGHZ zQ9&hs%K9Bnp&=>V@tk-czklxR0WpJ9s}9-tVb-fUEeF-S5io^}xsCDk z!rqnPhH?e4t)&f#k;l-D=TcagRLmz)xYn^WS*#Ip4N$iB(4rh;A}z>4S)I#uCFqlS z*r_tkEcn~)=ZK*#cG$S4FS1Z=!uYE&SO2W^CLzAdphhTXXeu7Rtz=HLQ%{xGdF!AE zpqnl&f-804xm)dS$<%asT;~_^s`Ou2M)`s>oM2;Yg1s#Y1lUz~11GwztYu z`-5G5H}AQu_-j2`6C~yR>qNRF3F_tf2lp@JZrD>LY7)KJ%0K9PXiLHx=|-Kog&aj5 z6xKxs!jHl620CZIJ%`b~v{Pe%@pIiaPB5$3xd0}pUQ#TNhVnSJ z+?bNrF67Ih{Hv3}hRPWz8?1Dx_TdUHJMgvy&RWBL?OXhoc`Nrf8Tpp-vZP}^_J&RU3l+ zER>UXe^`77a0?|YNBvN<+f!l#n}s294ws`6T(REZR+HRYR3K~06vsGJ`&)v)1R+~7 zuMpY{K_b==UXE6;^}2#fjGN6NTw+i0(}+Gnt{HBSjjGpSI?|qX3EOwAWo26zO&u>dOF09@--^a8k!Ow6{Xb_WfE~COL%#OF5`zV!DPJs$Sbeh0Y zlz@68Eqh>-lhQIq9*LdJ)wU5G{9Ew+(KBjHE4#pzE+WRub>}Z@q1lRxH+c(8@i|I0jq*Mj24BgRV~2?eWmETjO_$S%t(V zYJHfuU``h~^t{I*fy&#JJlRu=QdYEHE8A=gOmjI4Pu2nw`|iik!VGRAvG?HWy~ z9^qMz<#yEv^b~RpT8~TM}5R6NF5e z?-W@;X6Fj3Sw&618(vV@<^u77=h*E%9HOWUs(-?4cYr6u0Nr*BVzhN9L8=)1b)u$5 zQ^D`M$dBxmTv?`sTc-#+tCF!sS1g`^sumvSZ@iB^oz_`Ks*7_@amZ$wDb6mvoI(da zq`q<00!NN^lDead-03}k;6IO_sOn~wmzJLsBNDX3cqBTitrsPQOxySSMN8Cyi0oVXEgcU;6e-Y3#V(){GJumJ;!jAdGDySK z(mV1p074^+@EQ~;1AlX{HVgy03VWDz(^(@({Z%tUnD(U|3s(8DsOOCn(cSUld8NeN zUsXOQfN*SwB@qPe-mpca&ecsOvbU9GoTPMG&t)7$1%XvmA$zpAawAVwhyfxV8hVv8 zFWCWf-3K9NMqTi&d;1M)%`qZtj-X|^E==R2S1;y3U1ouuiE$Op_cp~ChF>Tf3_0u03{3ufc$ zqY`IJ>oPohF-fE?piFyR%b?>54n=Bg60c?aGR=@#01NutPBU$(v0HkU_qVGYaQHjv zr7?2yC4+BroQ9~AB@j7lD|W_G&4}hqUSw=+G3Z*uZ-(fg)oaS=iQ2^|>W$Rc;)Knj zP04Si>6Du58Zh`35}!SzIZGU!p@?@awDgIF#eeVG%fq3`-Km+7fO7#mn;Ddg{4QHJ z9Z!-)Nzk8AS!IwFl}F2pa8Z=#HAoW9G<(cO*&0M})M2}a&TSX5_{VJVz2B#(1x1V; zA*+U%JdzSPA^STmyFlQYB<>_qPOHZnMCCX+Kq`1eE*1ZUbqKI42k9De#R<_jG-E+K z2bFdHT+}s8a9!QWZ2f~@)>WY;%~$|Rh^?Yt6fy(zWO`1-fsNQ;>R%)>fvBF3NyxDn zR0dVu;S3VKY1Gv4jhWQN@cqM)oYL)B#g7~bxUvrd(L6-i0&Cd<^aNp}4;=A0See}` zGR)#RQHVJQhgoukQoE<6e+!AOKmvaLt}{A4zz}7eWV{evA`cn#OFr`ev20P{&)Cgp zINOvx3OqHQf<{{CK9PkvihORkkiep7jZu@BJe-Rf zMM2nFmwCF-OKh9hccUI}Kn28-(uIp%KGq_t^e_*%Qm8a6;t%#MCJj=y1V7Vs`XBnn=r^jMWaPfB2Z);|g{^(hX zW>GVWctNfNg>A_%;ToK694hZJ@f$3s?MyMQB9o2na%u#_JbRHI7a0bI*h5uec4BYHjKDXSm_P9! zd5euUvtK_)82F+5mK`~dOO23bD$acsF}_ot%3+BAt%UIZRziI4{|A-OGS1&h$P*@N zas(;q|KjW&qa$n7Ztd8%(Xs8M!;Woqtd4E7W81cEr(@f;(J@Z-v!9K3pZARMjqg`g zs*+S?tr{8kystUu>PaOCUWsZLlprbPsx1hP53=C`_FD_=Dcl@b+x}(`DGH8Bd(lbW z#V5>E*)D3z1)14bxy?aVFPUD(@#Sb(aj^G3Qb;QvPlpFgWDo=>`E+94%)TyIRiRi6hIs0UazYym+IQyhu(A-mM+}n%aM_1 z6vNBj7>mqBMrO2xDdR~4oaHREo$EkYWC(ioNMM@su}`m;*b)XaUDP}_doh?oz5bax znU64k>aYQheu&miw5KROYc97S>L^|mRxeJPdYkozf|hH; z@2xcCjUOc}<1KT$5iv=Ar7JKxw2>Xe%=V2Y>YZFKZ9zD&x}SWH!=BBLJz;^t4DvZc z3U@GTuF_7URCZ}ubqFD8{L>*{zk{io!_>#+=Fo8qF3(Tb4ys~AAzjmVXRAw-Ee$MI zzPc&^>VFWJtfB8P;Krd8KOTdM@zO3twPGWFB!2kNrzcO0{VL38vsHPe_dgO_E)r}d zZo9zSkXORZqZ!MXKB2KB#11q>G=_^t3hPvrPc5v+p7FSN8vG`at`IL5bDKE#G^#Ld z2PEVPx;YxbK~Yr&yX~8mO=SG74efp~0Iw))(tS63-t|nD{GLS-Cv1zET_zmXu#|;2 zZ3?-0(@>bfa7{cxL>GyT&A@*C(E^>ZA3Lm(b35vYtPE}N4X4B2A`v+|8r8(WS+^^P zFtWfA&GB3UQ2t}t^;bPC`1^ro#(*{=2jQ-q<(lBt8#l!ainC+L0Uzq0kddpGnK&D3SJN%yP!&DD<^0udR)HjOk#cn3V6M z%v|-365ii52u2FN@%$>5vi+vNxN(X=css93EoZg!Tc$PDl30O`v}}SR;4Hlky4>Ci zRg2>yuWoZe=1hA~`AjO2;Nl2LCag=xo7qRQtGna4QC~6+aowS@{B=B^TgOvAOU%bp zUN+XsY?Ehgb^uQbo!iyTFk3Wh1ONjM;8aq>V_w@&vpK-_Pc79YcK7Z)A#q`6Q zZ=)i7-TB9GL*WS+ZnCcd!_89hMXd%mqSev@t#+QK?qC8jSiU1WHdbLh^yA4x19!?= zNhz~GzI_IJWhr3CWjen5onm#EJVG-!l+Jf{#KGcrr3s5dnltt|kOS}^v^ymKl3bIP zHwG`~;=dLL7IZzGXv>-Uy{~)EKL!{nVvo*-;*IkO%+Wpl`0!nRTcN73nOOIIk6jXg z-jN~iP*xV@QmO6wl{$t)@#UbDE|%L#69zP>m+-=b!*noY_zE=KFE4&%jx@s4UTT_0 zH}=qU7|cQ5`KkvGZQ%p#FlYaQbfHE+ArCBEo|sKN(AfZabm37CqpVt^WSDw>O}HG1 zlculoMGw$fpD!nx`@bMcEpDQIVk!B(|Z=~d#(T4(!UJbH^gwLx5-1mH4ac`nskfq}zHf&g#V?Q92c!{nn zl>DlG&;=CimKqgB!tcv4m`zF1B;L}F+lG4~kH@E}Y1(^btC!^T<6!h$6|g$Z6|(j= zs$dwwu=*TD$F~eyU<6zhQUSPBX$TeY6A&@QlsO@r!eJZiviUFBjLu#L2!}OK_imI+7*mQ_Vs~|*0r~? zj2Fvz!YQTv7RcLonrMxU97EzzioT?E7M?g6CiZ0QIvQ}Riy(`ZN@2NPXUg|V5dLxTm|OJ!X0gho*HDkyF@$p{<;QPUy;;i;9z528=mKVzCS{N2^LvH_8x5MW#$hAY?}v9Q;$CxVp& z20SSPUw?gWROlzd8w5B#Ofex8)gy4iJ-Ccu`>})}$DX?UGb7NFUg^sT)`f zB?!v5{dhr;L(9H2(y-wF?Mh*QB7m?!0E+<=(F*837XBHC<2Vh}G*eak*J&o+?E_zedMIGNQ38Qz@Bv=}*it~d zz1rsDSs=&!>q;BJq}wA_<&hE)fJz|PYKubK^2w;`w{&?(dIat2?|`;ZTuFz z7=!FOUNyD%y)+Z@y{v=G@c?ELcm#Ybgk0SAfXI}BMukW4*nufSfh0i^5)?raq~lev zB&#ysK#~COMx7b(Zqya+LEvl^Wua^pAAm9d$A(h_aBMhRZ<`h=09Se^lnp$C+(S(U z{iFkA;MfuPPB9sRTv6EB0m`Y1 z(POcnR5sARKSUR~-X#0sW{GvG-fgxzVbhh;3}NPd7ZlkDAQE_2pVpn9ue2Y!EI5Ft zuy(VTUvEU*llKl~z}Oh!J8lk*hkp5rU3Dz!5AcX4qmC>Dpgv|Gf07)6k1NXQC%yck z6TO!ST@|TIm`56kJoJ@Nx{$_@{GPqYJl4dCTfOAFG4!D^)3H+Th&Cdi4r3Hrpgauluc*zh#*D=_u10wZ!9n-73iB}jMxH64tWW)9#A*8%r$NoNqc?<54OVEFfCwSLX^jD-v+NTirgNT{ovHdoMWePkYmh z!Lp@R)sb8joa$dGn`G2q%-6#Y#5A@P|D|DzkCuZ^5S_4qz1k z#tZV20umjO>Q~hWl(w&5d?=)S8Js%2N1*0umLg8MXDfK>(KvGsCv2oG;TQL3w>rUL z-rv_0Gc;$TP`(r{49#K$Q zK}Now?YI30wQqYO`jePZqhuWGA&`>Adv1|^)BXurMn=5`&P+s-x)1Ga&em;xkrt0P zS;Do!N@4twc4pkHbhXks#@;AFp}a#Xp9jT;A2rzPbEDucuLt^3X6*iJScLP6vJipC zIRS$%TaGaE2LFkbRuS|oozt+q#8*B6FHQCapDX?WU|7r2xkwn;^DOcWFl(8l(3Zye zULmqpr*4Mg&8o>YY2!1PpjfKsjCzR()tdodRVcD8Hp7S7s!xJ()uuF! zXfa8M(L`WA^(@C6XiZW#Mw==3Hy8Rq;Qn+Rr+eYK zZB`hn$`r_g|7j?T13dedL;K||8GH@JKBiWmJ}E%2Ka)%hPww69(}mMl+7AZ(MssVB zM&-K`QTd11IFUX6?c|le=Lv-Vq%OQOnO!Wfp%n8`=qjbT<34oeS;RTEvv?Has1;Lq zAQMC&TA*X$au-yomDw7a+}f<@e1^IOtWlGlBXLDOQ9(M=?0h=bIepx87|bsX5Eaxd z=hBo#?fm)-GnpOL>-0E{LYW2JSV6ntO3mvb1#m_oLjP3y;~f8?^jnYqR{DAW zRQePDrS!wTaV2+MG$xB>j!=F!iZme_f7R`!!%5XsJ6b}vbi%;dmQ#E6;6<79n3T3EXn`tqjSvsa=dY>Kknblkivt5>y zYPiUffQ0~LzLHhL+QoxLH^ax8b#wB^8XvCX+7$Qu-R5%+;Bknl)s4G5omXsgn?2AL zgrRed*BL81#47UWq3Fu$Xm28N{S;Jqw>G?Y5|`l59n^`xR6bjw=V4TOnQ=qhLu*QI zQl;|*)xrN?eNNgL3NH)4zznxk@Ycmf(l#S+nqpTxh)qlfs#K5Ofy5Ih!W0``!q$H} zF|$NBul{uVx=wNzAO%uSj?*RJ_aQhzW4Xt)Kvg_o<}OK2Wd0#Q@*?k6IgfT2EF`Gq z=wDNsYZzWh*)a`L|K@GLl=vN0tNGH<*wsG@-(Vf(kc#^s`kZ2H!a2WET(m**lOLeY z4~_Q}WwlEpyR}ZOdX%cauuG^XMAgUyr2RZ@a&Q!uu?nf8br349uJJNLit6NhyoMUY z71p1EaZ0wK)=3=|UGBun41=2##%>|1qIOL^)PPqZvwy-Y;d*nNMHO*N|EF9yL8K!LwSU2!N8?|cZekBrRWBzSywXm zTK&neok|C? z*bj1vQ=>IrW$QHk54*~w1?L&$b+TcVE)g6@Bgd6Fr%Hp18&quhN~8x(b7oEHRtKyI z7b7_Ox9h}MJ9lou>On$F>a>conN-SUM&fVf%DBa5j%L?ARj;X8T^y+hiQ=|&( zzAdiLPMfs;Ry=1HxqOWzW4h0RF`ApAzhE$c&Gd&Y&}Z%sLYos;OU}&>GgeH#8XC0) ztBn9iM?~H?mXBXH%QwCuf~|tmIuohg;uD@WHYPg75F5Q6pXY2^?;2N*N^_gl#|<7Y zYw4x;wAceR%kRhNOkJjd-}RkDS+`_ZO?Umdrpit-n=#N-<2YWcMYI>xegjem^N7^k zzm3Vhrt~R>eT-0avuk#zZ+2&1K%yp0Kg&&mv}*tC*hkA8?_Eru*1AA&Q6CRoIWUZYNpWH~SFSj=9k{2TB|q!62~wf=y=oHrRGr z8_h(q6B?lY6-?-q?T=q?kbV6}Fu_T!?nU>DgS;q- zxBY8LBLCrxSomE(ku~w!EEp(b^qiJvxz7s0JQ4ON6Yb=+9#jplpvD;%$O5$J2(_hMyU&^yzERd{XOG;y>PB1z&*X$BATe_^jYcN!m`rsUWKK2MD ztgmb7=S56x#-SYK;p=4mGGiYUW&IHE@7-0_AMikaq_HMyv zsl)CZ%!v5s!eV_X1b4mRrwM7brsLbH4f6Yt4RR;h+kU9a9ZI`Uk9;xuI-q!2ZN%9C zTBANz{E^p=$FIq~Oce;vxV%67TA*IrMMf#}-G&SHA_}7BMy~tUN7F2!iIT}uf96s# z2;G<|{GP|X6o?N{R4H7Pgq5tKyosAwtPsl>Y6sYl+4exRB8hCl3P_8uS-)n`yQ<&q zOI)sQS)tsY4B710${CQ0`4GRGx+2rm>~=&qW|wgi1$yDg<3pd`Rz|YSKF)E=ci7)gEz#-=bG!{?Ps(J))ycm5TGwWiy=Vb z4G>H~NT4y@o{MxadNhEnfe<9Z?eQ(vQc{!MnVW~LTh!E= zm*=8c{Mcq{Q5JHg%jVD#wV8_tcDv)+EP*LU^aqu94V~3s!pG|DB4E8Wq(BNu!Lm?0 zk#v_)y4X_27>es6Teb-B>TJn0imGV+a=S%*1DllnGJ=GMXoQdGnq3@*Ae*QA@ymEB zGwloJ(c6z-0RtG(e3%<}u0UcG0(iJWc(Is-UniGcLZ!s_Lq=;eUu2t6+Y@df~{qD!1582xD<+A7mV4k0Yn6^B4S-$s=` z4Yg%Z=Ao;6nVvd{u4AyiaPJ$I@lN_x9^CuI>CumS-!$Mux2dr5(A!J({Br@f0dnkF zcdZlQ$;WC>W9)ORiS_yoWQyVXO|$=QY>hf#3iHX+m_Bv;0(?`ak+WT=?dNJU*&t>@ z03?=(0zYloOATI6*kB2F(aVrKt%`AlO6EBoceIKzb=KxqDK*IkP0`wj2(KTD<~(nP zka4!qAYo}(lv(H|mD)^quf&Bd@ys0lj19;gZRQezfB34PyS|F?*3Ev@Hx=yT$s&6# zg6}HN({krY%8B}mA=%EI_**$*Sm(nWZ6wvcF3Vk((@P`a9tx-K8U1USS`pE#yCp+W z7PJ zR-A;pO3zRiJBzyLSaneEo7bP-=I_)$*|s5JymPi8(=H&}X6#;~e)mGX^Ru%z&5iql z0OoGmp+vPUnWZM?ZK%J-1Vy2HWn&$cEwx z8gi8Aw1=96Ro9~0#RD&lLJ}@>tb9~SNpmSK6g^fhM9FGpYaNVFFz!sr;_Yi!EtN;G zSVo4li0@qAObujPAA7kkKqM8bMzKs+;_n9B-o$qpFZ5 zu*t$RM7FZViBpV+!9Ni|x zlXGB|lRj_ksS(w#E)V!J%IrTDQi?2^vUC(`ebdqP65OV1IOO}&o356Ny4Vj7*CS)& z;3#=FQQ)doMck;T#NX`cpFA<<^x+wSyrz)2@cbATz=TcX2#KB5H0 zQR4?&-T3m#H6WmY+O4Ey()u7yf$EKC{Y6&csT#Svc|_v-9py`JQ-i)En%RaMiJr*0 z#v(h{PbYW&z)#sO%*<==`wyC6RMV0ty~7%7R`i`%G8B`11wD6q1;+0tT|?B(9V_Gd z!zJ>os!<8gZ*!g>X0)f_NYxy(;mw;6 zL7>S)3)Aqv{d&RoAc*=mdhKQCrIET+c_KwiBt03-mG{7rOjiEzn9K$1 zB$Ara?70xHyQJ?qT{pV^S@fK4oKQVg*j+Iz<=E60B4gY10`L79ib_5X^DWS(f+y6D zh;7-$?@O|j4;SB+QfJ{OdP04DgNe2Qe+`uK=ofh-q+wxDJtUw20f_(sf^0v*oDT@U zyxJ;LgW9mb-5%w$Xvo}V9pf{${w$TN%ky=dWnEbc<0ihr)R)c&#F zO3#V3JkFa^gy{NKsB9AoS5UF(Lo%0-oQR`-jxM=^;prd%eSj1D zT3q;#1s63%9c;p?t41ASh98jT}4>AVR2DKXy^9K$bG@&9~|?G5rgMuaspMAX`^7Tgi2VS5a4jWBkC) z%>vcFI&KD?ilgH?6Tg-BWkIVTt8}x^q%=nL**0jb8Cui)Re>vS1u7L!1%P+E{f+Xu z<~`6p(aDwR`p;sm0M6dLK_f(waC8*m26bXKk?;#rO;}FNZ)mEBgIrcE(Re7Jp|!-$vz16Bz!xVN=mim@^W_ZVO+z{WrGO@t`JOZB zAwZcCc|;V%y%<%hE-$ z#|8oTsUS+~IFO2NmGAGz#NRYoVi0I=9|zIEi72Wq!maz(AM-=otaM+VD zi9q!Yq56K^ju2VWrX%(Jsw*Y7q&-^Muk`9t6tM}BJf;(5s4abFhW(%dgm`wHH2SgX zk{=#A5`;(GhH6f*5(18WkLNj4EdO7jp9sn|zwJN^1jrTOl~TL!EA&RDZZm>Q+zQozHw7lR1UVrn0dRQZj0gQF9o{L(aW3!C!g9uuKGF zo;SD)aJ7R_to|384zIk@Zhrz`>HXjCXfM#7yw7^xk+i6lEsLY;<}Q0GuRdfO=7zy{6|755H8dxSs^9rC zrFW9Zqsh|L)ht<&PX%;P&v-b$DnC(hlXH7RId|6e5$~FQd!vvV=n7;LUi<<0%KwZB zNJoPhsAI*6mug#^#s!sV6Oi25jWmZGfDLicMZWrz+$ldYbK{iSGFvE;@?RRq{do_<8E(VGROCjwcCC&$n7fG{RB8L{w zg=2@Vk$7@a+aka5<@wS36tls@_*JBl-|wauVhb$*17aZ1A2jbI(obV!;f0W$e+N6c7frgS_ODDtnJoV^N%!Mx&cE;h9h%M2 zfOKEz9OJR6IrS_C7n3EvgSo64YeSyWPeAGAcV!e;_Mw(u06h@=n;ukcrA^mM?VqV@ zB%~r*v)qf5W{t8PVs%ciCZVa0%hMqPQ9dVJL$Y#P&r{LnPA|=OR3i){X8=$bwj4h3AZlI_L%(4l z8cUBkm``60YZTX%DcPKT{JsUjH*SVUErF9~5DrwUIfyI)#*#xwmSv znk0On-It&+3d}J4u=@)Sv3V}dSdF&qH16)1$JK_9YGTPQj4mFy%jC83D*|cxSel{;%I#0F#4~o$FkmK+H z*O+a)KC|~fq7(6r|Bgm&F7s}Mx`DORx(qJEbqT_O=Qin{LNXhl4?E1s$EAc@3=+Z3g8%W)qc zN?U{++=BiwY&tK9T87G+JH7+p^|_&iXN&l3#p1<>6W5e@W&Gm@wWUe=E#wD}mDn?p z!QA~3JamoldmD_`}!?=Bf#s7v8il6Hw9uyts{{0B7o_-(;L1}8szYNQiWJBlf z@0A4R-~2@Hb2qh0sT?K#gGL;WPSdyl2aVu!|C>gfi2v;tr2Svqg7UFGm>RLs019C* z?6}6&mRrOwqL##!JEyk%l^JSiGs_E*op>*((Z^(;jX3Ufe%s9}m?#;c5Pu#u^E{wD z=Dg*L0~E&pfe%4{-~&|x`~MCfQ2)S(A+O16fCqTpJsQ3RYm4D&zCg75#ASuk1ehN9^r$K?Yim94#Z zJZ~@$nbkAFQ7j(w_HVafF$O~$z%AJS$1ND$j1dYw$u&q@+x!LRKv7Qjk6VyS_@PsG z^P3JIG!QN0&0pCGlq0Fsc(YqQ_FzjX9@iiZm-2KSsNJkMU4ldXHSo_5Ji6&EY4JT~v2 z5eFz968(Z_vau)4zi&jg3r4Il4!tTdJ#y0h}PNEa9i{# zFfD0aI&Z5_2Lw-n zONxk@Z1{|ld22z==s&X~V~@X4leH@t$xqqS)UMXBx@pPQk-YJ?Kz1#SUcV2Q_Vu2db>{+RifX>r;yQlu&snsJ>Q zL5vAz_w+paie>xyun(J6fmDyWNH}JOvD3a8MsXDCWeaa@jTW-@ocV{2a~TQG7FCZY zhxuH}V{Fe+x9SIw-zrW}=sLbjqU9F#*Is|8YVBVQd2~$8Vq!){Fp(^c@2_kh5MG{| z?Du{M8IsC;x(FL0=?Zz~3S`JVZsGJC3XAVD|>7o$H zRx|YHZdy9ws>KesgoHw5lKGI*b;RLnPlD!=(!1i9?<-V*4d%{p({)mEnv%tUdQXrt zvg9D3&Fi6w1+BjF?Nb17Ne>VgB58@2Zze|Wa>5U!euKw8Zn1xwCrKNq$gk>%vaQ1H z`UbwS{4GXyU%~+{SwQ^?B_mIi<+v$8I#+IAC51I=ovNP`KMa?Y`*_?4-ss4K%7uOo z_2J3{JKg4-$HzQ46SCxrn%Q^}SE_L<4|~2&k2gDISc!YTIu0RMgYgS|Xkim)Fb!wX zWiy4q8{*6C+Wv|WgExiK;x_~cTHA5?EyLhErQoj_%so{7fCLRlGfFX>N*$9B%(@X- z42Vmsc#v=gF*toe58V%1l;k+}=#p)MFj%9>5(kX$TjX`dc77r)e>p1IqyyWPLj9zJ zxByEImKa!$>Jj4+KgX6oZdf)p>7VWKi4e@Y7yvCdARxNlK~79UZ!{1T(GqH!J?URo}9D?B{o6tZOd851o8r? z$%0_CUN#tjp|^o6@ajlG`01mn?~z!_HVR~I?)tq$z!+*v45=$y4r1qZB@l2&27c$2 zYYM`vrpmK0WD_ZITFYO@K5wJ237pN`H=t5^@H31|VoZ z*}omZAA)Jb+s@2PTb!@;3fR^G+F@+Xu{Le41Z?}*eg>TTp3ifQcY}l@w3#G9$_V;0 zk06Tpnq%GQ#{R48B8m&BiBcHpRad<0ZrKOWb&))7qFIgyWNa8m{GrQ>^odwbH1Ns5 z54dyl$whXG_33lo@{0+OZrfX4TPIydJw(2n0QYw65$DuYCKc7Y83NRvo&Bb~?|*jP zeXYnxFLp{Db0_0noclPXfV{}n8OeZb?dt_q0u&>Q4=G8Bk=@}WXmVR_i9_rH@5XBV z_^cyJ(Bc*ND)1G71<9xKK=^J-T|inu+8*{o7wFS{3~Cs97+NGgxol4&e?%DaMAuW0 zufk)KxB{XE*UjM*t9!a(9cg^je5{%>}jxJt62}aKwZ8Q+bIB^eeHZ2_E)E)~I zW3{8NTNVV=-WV*We&@CQHE~I!_U_=Pmweh5Z)u@FpbkSygv(|6F<@lf zS6G^v2FOVw8-Sb~_CSciB)S5~NjKnI2!Nb)>t(@WyfP>5>1ZG5urQ_uolf820qRt^ z0CESuzoa?VcGIe!`^`&-NO9_6snY^<2a5yF(7{rICRV*sTMtfXCF;&3sp zr7*q&K%t;kSte5W~z+O672dQTV%+#|Bu$T7j{wZjPXJ{qr z3GM8N^J0_RSMv1&IY-sYj#2RU`XbY0@d{=n3;_y`MVZ%DZj(qofWF}jU;$qjr@Lj!i)40Ehz zpU(%h%}NZUw*m^(;)?}jOYDRXeEmoC_x=2qBub?C}#7}@i zhDWen9Qpyc9sAw`8nI2#Di)qHxMltd7F!hqiuiFo@97_Jy55wf2T zs2?a8VBTef)8iysfn_NI3Z?@+-Kd98`KR~s{I?$6h&U!3p%Ca%#pSC)H|%obq#2y>0p3ML zU#$LRj1&IL7zaD@jSScfa5i5u?84C^yZ?x~ z@k)JbiJ}Z*haarts~XoJV18g2)uq>e6w=1@!ViiKZB`pyL<2$?tZZtOfsUcE0=dLI0(sJr*05m))b(np-fsnPz646BPCg|54wJ++ZU97ZV^0b zU}^qhQ3@OhNN3x)-W#I{`M;T|1c;K)7XYV;hmSc|#)p&L85M=7D5pZlE=LiR9}idx zJ#NUqaujuUaxC~y5&RgTrC%nYOLa%p!o-kZl>$ek-&qc|fmi*94pyNSQgiR7N>y1^ z?nH((ltPC@#n1E`wsIa7IBj51E0L4~^a12l{<}WF{Xj?Ur76F#@p|aMMgpE$66Sf& zo%U)qwZr0;z~%LJNilrlKXY?YXo=6B;MC?u{@R(kV&XU20?~?MY_)|ay0PyNjnj$7 z%k&h-@Hc8qR~rnubwWe>)S?un@n~vK(v(lTp3K%3$(d4lP0$3=lMwGZ#cLFqER9=( z0f5}x@clmF6>ZevohXn}c5l7kPkp6Kc>8b#hgBF$+W|B1@!7Bo=t$Z6Ih&2| zoRK`{=8KtgP>WR%4k@d)__F{}oMoNAVck6S;&rp-+}?S=HJP(@!hd!@<#7}M5X-o{ z7A{k-rkVeDn?M%te@Nm=*<(UCK$}3e$*}o@vWAN+Bn@6`i251#WkdvH4oY|(#%G`9GzWSCI!*r27Ep&XJuvHZ7L}${pmb;A zN_rw=*7PF^Etzr+dE&5Bc z;J>YPKdz_4n*SkYVKY!H9{*Vb;P7`1fP!@suGKf@Je8NvRzL{=18%bYY2U#?d*LR; z_-|^1_&}9gs5RCSw-9R{F^>89V1P-I!tn)WO$=gZSy;ZG1nnw0NQPBe`@^n(RGGc+ zau5SyCjn)iq^t&;?HPU=yVvwOL%0R8T&zRau?`QazewEt*}Sg zQA1a{ID*yu6XU{_XR^)9*syeMgI5M=8pHiEh>XzHx?V zO0SVYoh7LnoLipzWKqkhQWRlcO*ivP8apeS+GVt6jbG&}z=2n4Uk$ewuC(L3#wFvr z0wawg{NlE+%A;m`%t8~TgpPhz_Nah{&+~>E?vbhZHH#X@$r69-H7hRc^FR*0mJvF9 zYWh(@`{~7I%kxs8V_HjrYP-Z(;DvO_s8~o(F%2bf_6Z{3IGKoXGzufS+MWax2 zSfMs5^_tjI0Gxti^dt^3l{s5!Zf99}#2}aUYgRCkNUQrh?h743)Z_D8^uAwa&yW4D_`~!yn*`p*TKs^eBntK3)>AE1-lLGDHmjFJ+3Q$=&Uo~sl;6_ zK(w30JkpYvY*OA;(I)I{)8O(&=iBkq1%A_vT&3=eMRjI8xBbqllvgp~wE>|kQ#K~s z={haLLq48PMDk@~n|;z58|PCRH^*_Y?pYP5?y78SV%r+Dx)V@E+Ef|&4wbCacamtO zrR`Q-D7jl)ZduM>ON#nQ+g*AyQOj3w%hf6G89%hWBrjpt?JP#zEEpCx!hM0K{D9-( zj$495_Fz&EMf~F_lqn`V`$L%kjSFP`@O0g3oO(c1px(TZnV@f=+s4UPfR0yNl+U3u z$mS*WK3xw2*B_ru(b~5E?0Bs|=h+OTEzbztnRiWvKDZy3py=a^1)HSJeA zgm;nWnWOHkmrWD+?pYex&bw@8J|TaSO`8l*O&-Cu!ufN%%BwA zwzuh>Mt!w5EBTC^4;xRlS5z=F`H$Rh&GVECpG}#s5~cuuqVk3EKy0WgK2yTj*tZySQs6Rnd+*?J&ElY zefu|mSEg4lrGtKhHLdQM3^%mhvOVp2f_qHdyr;ym5mK%(ZxK+>5yO5t-a zlE|WEyIS$3^=9ZOMV8d&fP03QUV634G?OW&(n$quQQG6@@lA54Kl!H`&|>7$R7{2( zBGTVY^EhC|Igw%cQ=e`u6E(5Wtn{Zc3Kd9>YSEsRWZY)vdLGPkNtsb_&TmyP%;S_7 zTs_`fXgrpnZt<5zdA72XMy=;>Py5tgD&PYB^4R@Ev)q?OrqcVbc3@?)5@?5`Oe7h~zRRN5uYf@~cw@i>7OhUW4zJ1%N3~w5 zrBr6HkqjhiTeIax1UKcsp)Cy4(D}gLgg8pDh@YQf>F`}AAR|Q!D=mfTXGu~ipQB}K zfbdpM-qlOX;?i54-)KzfEO*(|?~M{6{w~;Qi)Sf!uHsTQ)!XN{#aV6PQwfzjm5vTf zhvPvP=<{X?hC_HuO2m~`u6@aBWWBLSH;(FnzI<-A6z0&rJMFCAJv3ff!d<(UCx+yd zwU~*-ImY;mDau|(RDP);Y)P$}-%u_@O`PiquU10_O}roTH8h9nui=HUD%GoFEcTn- z$I)whH#r<%ujGE2yHrF?O||o>oHY+^t7sJ;>3?+hPJC6g*;=4j`UFJk5huB&KDs*v z+YvtiAH}*ipLcAX+Xv&Ps*~5{w!d@nfFi-M2lKk+I$<7saO9auelpy|2+>$I$V}T#2^b5=>yT9gkf)M((Px~mhqZb? zV9c|4N9$W^)N18!MDfGk^?hF%TM)%uP6{)E_C!+ik=d-08H~qeHY=EM{+O9pZrKzh z(!f~uzjrHsdH!+U+mS)5o|+aYW!-&qb>WV=A(kXgM9Y4ymNb%DC}kaWUu~dWCSz}5 zVxN;6aIYx2yx_TGL3Sy-CS4a-G?bLk@wOk`@i5oZBs0+gH~NjO;IfORg@&%<11=h0 z%EfI5{RaZYL;Qy7;aXCRf%}tqRzauq=<70NB3mqramXY~o!AzY@ch)%g@dg3`77mu zmBJ&~k`#*jkzLfebvY-W{UV!Wl6CF%gZh?Z+>9G>A2FE;m?`qd33rCHM%QXO(ZYRf3l%2j{C>8Yp|qNPl+`Z9<;mJ{8*93gZ$WvbKrdW4L^FE=HoCI z^{STn`)da<-??@u?sT>};q~gQ9%9W~4}pND8{GpCX8H=Kc$zOC&^a)SLppLGW+NJg@iceI<$nm5>8-zuju2ciL4^0=!i?%d=8%nZn ze_n>SoqIjbr5aIS8d7c+OM#0IN+o{7FFsL3`R7!@C^~Vj`8|B9468a?ZDhEEygS+mE%#+U; z7pa`9cxCpboT=Ce4askmqrDevSYA_8lrGI~qeMtxVd8WLHP=A3A2mmxP9THXPJIM5 zbO|~MjMGfRd2^@6s>R|0b||q0RJ&_siw~Rivw%qbeOV}NidP(nDf6E0I?UnTT61s$ z(!kNa;P+}F&t@zou+(3|P?m^(2($GDoE(RurS>p`dxa$-SGOuU6e+P#$K2d$Pjg@| zRkhAt2UFB32Eryi3XxL^))DX~ic}4V=*VOZx5)lV^}{#n8Lse0Rw9WANi@&XQe5~I zA`{4-Bp0>I0NnjcVrA4D9=SNF+FL|kQTDUbt3jqAyEOGNH_HIqw2BHtfPZv;&E~+k zaP{1vYf_VbHb*QI&y7Z&8d_jM%^mF{hM*sV14IH#J+8hJ@u5>ZJ%)^xU~mmPbAMQ+ z0rZ&-kIOxvaGH7YGp@0ZlVv9n$~U#~s_Y)j`L$jgD<}Y}@Im zV;ddYwr%@XpL6D&ne)!9b?=A!p+3~EwfCy0u=bDt=LaR>cdsqy*bO4zIF$;pBZKGS zto#t|$r!e+Ub5~GH0WzuH(xYH87M*_$J(u9C7G!Rzc6N`(pBZd8C1iQMSe>#2RRoV-{^5PtwDOuspU>uO9lf#)+H z!21l}n|BGRY*hoyTjH_Ak00O-cu}WPC%LNuWww?Qy=CkSr*pWmRR-6b1NpX;{6JY! z{eVI0T6PI6?PHDI&yF>GI!nnsj(U*vE+z+WcvynT<^!Pc|Fn#f#BOE#pmYbI4+ga8 z{;={FI)(rNK!U)7Vrc`f-ENexA|umfI3P9e+F8-8jpj1da13ZTSRDn}x#>dnXl zkp<0MJ2a{n_Z#kb!v~PjAd)%8pffB8#?M$emIafVDU)nsvv)C&zkWj30g^#!A=Gi? zf80dQY-9A!{sc3|o2-bxh}oDpLxZrue0K}Ja#t#diX+e!h*nWaig2_YX!V9JuQYwx9L;>h4;w|X`z%7kaoG zuJOUT>*%*yg26GTCjmY?0iGA`_s~amDVtz^lNbJzfqc_1>ui28cfw0$AOzji1~A>! zek%Bh_kvl>qYVX8&WqQCze#CJ%_lB;W#V1Qu;3x!F(%Or?L0sMhM$wL_#i}oR}fml z89L+)ss;B=f}t7`4TKU@zVCYvPG0EE_0_kle3wD|*nBPCPXW)%jkU(8`A|vzJ#7ak#kHpXuQ0$alb*&kdi!Ti+7F19IsbjRRSJ zpq>Ew_O}2zNQj6oXx^#oaA5qfFgaXWU7&Yv$>#=s;EOo`WKQ zb;3VA&~a{^xwTK!S_s;SZ3Jd)cB7>|bS#FDO=x`xHffU0BF!dD+9@Go-od=M3vgE* zY*WTi@Es zPx|EBh}(fU_C9dkL|SYaf?bfjXk^JCNE#$K00Tf@-NBBUAJSwkAH18J$5r)R+|5P} zzsu23xJ`9n4`R1|NYKwoe4bP(VZR8qO=utw&5=R)9FZEeLZln!d#*XB^%*AjJKMlW2w9_1Ee_s?`yP{x$ zaSatH-SAB!^egVjHMS>7*YeOimxPCspJkEvc_3pBq?C=iy31dh4CZ~Ww^>5} zxIt$Gb!j}Ap*?V3)R{F8=@@K(TCeuezBPGnGqfF`(l*?r&~=WDY?i+{+SlFW0Dnj{ zpD5=slm#d;_x*Oy4BjfAB9(Ba8n(&zlRhxOv0rd<`?dU^YCSmp4-D)&_PtBz`RCPG z9a&1-hOJZBBm^G;>Z#hxhOQ9;H?MuvEwMV7w|jQ=CDW?B*h~SOuwVH*d;8*j;z$Ip z2nUVp^&x1HLbSPoIiG$fjWT)c=!>6!y!7Y9y&{SC$H)fWGTrdj8wQ3~8phg9H7#_6 z5qk!iwZUN7#}tVPi6r}zc&L?B;;$Z|YhLFD1=qPD&}pzKF+@nvkiPt;jj(p1RF)Y0 znM{m(nNr_`X4+Xo(!gDd2oDjI9fN5pGKKaHE`tOzB(~c$QVe$}o#_QnerT=wSR8-w zn7NAU<_O>+BWl=w5HMW$eg%ldBxX!=+UbgaG8s;p4cd0qMUNeBV0gFIP!XM`z8R^n zBmO- zHLfW-j^oy})ie`rQCIzFcCE}RHCHWSYBiV-X@ypHcC&nj7VJzLuFQ;nxBm5}LWB#p z)7Dwe#$U+{;387r*7`uV0LfB;HnSb*E{*lTBm~3SFG)YBV+N=7Tr+ z!%}O}s&poh6C$MUQ9BP5c#wQVHy02C6D0~cQ}Nul9apmv z&sJyB+Prv=UHcubjlAl3HSUGvo6t_+i2w7j0wb_#9OYow962bIE74BWZ8p!xDjfXl z@cc$%!wy#$(|aD<5rT&qSTWzxgU9)7kHuxPX8jx2p>{EW=y7Tiw6>dU1a#0kZ$C< zSg_z+xZt_lBRX=DEdvMd&J-24(gUw`+>=+79$?d1EPE5^6kk{OV++#}7|wjx8=G$@ zVn@_8I?}%MUImSsV{DJ^=MP)=OOygRv=y|R6}$z7wDaQ|V}i}7srVk0F0iG|&k_p9 zW%89U4dssrAPnw-N{cZc+auFgpm0nmh4Sj?eXbkd?;ZJ>BUDFTzz-he)YjH;_3u>H z@pGl%2vaG&&pt)7hC7OjqUMwdE9JK2G7--F%m@F6|ml)z~0S z_B$m}4_Zaheyb%7;nb;M&A;a~1h4gc|0xWw-U#uqE7TeD=C>&0V+TwGZ<35m*B?E< z;eLLB6w+)p{-8M>t0(C(o{GCOddLt}SU77U))ga( zY!5-Rib({{n6c?aIf7Pi=uGU4Y!LG)fhK5yA;*EU6;ik@SfCKBj%rmHJi0pqO}vyR z`cwifFgy*oXhISiY(x?&UgQCHp>iARzI6@SzBRkb3c+I7=PU*{Df2<~Q8cQE&>s8c z+$08WU!%Y^>Q4G>kgr-8_2QJy2+9xuUmkkvt z-vE^z)(neH$C1H2u0`c0m$k$fPXJ5Ta8+Q<(me8u==ohddPe^RJwRYqE39@;csq5& z=livMSkCaA4lB`2x0b|O-*ut%3(1E_Ar^tuw;7Z_;XeUW4jhiTEMiRM1#$UdnQrA> zK@<0()1f8x-z7|O^A$_HUguww4$sDpN6M&uFd&N^ogARgCYFhgfJ<$2_fWNT`q*=6 zyMm@qM@Va&b8vEu3w-}663114)M~Cyf)#}!nfqo&uwf!&*)%Ga93x4tOdfCJShh62 zwz`uuJXS{aj$dolXBp%^Q!5Z}r3Nz9@~mRwYh7^@g@PZ|(heVHAD?wGS#?W;C&kt= z=Qt_s3(tQ8O{eKKW)iMhB(48SvJCWh@?gim1smiIXevyg(4#1k})M*i)BSM zb{2UiV<4_TRZ+sGUc~-5bbIt8>x8mB%3-;jyF)vRpg~dEQVRpsgiZ0u?Vwg8N|x2C zK5<8_*=m#}d3+QE;TbvQoWWGUR}N+pW4z;GQBg8j`8iaNtrn$j!4r%2P&W0)?XXU- zW6PM{N%4Z~ccyf+#y@jd)TZr>erGFEywy^Nl#>}`P|;^1`m4nu-TZtJizg2j$C}<) zDzBsZ0pZUS%dRWH<2=;k&CL(!$)a~K z?oV5JR7El86V@C+(5Kr6gS{fFR%~{cDP^H98Ux+}fffH=Y+{uaY=9&E5%uGQJLa9Z z3tP(LlSfQM(bK2d5K6Ad75jzxmE+9$44Lu82pyn6R4{cpf1W!@i;k$Nd_G?&^EQJG zt9-4_Q6+KqypTwwhzpCk z@Qukh^e!+MK5cmD?@R93#?WT5Klu-W5!^Mbhl=YrrIO=Ecx$mE1W2 zj7oY$&E)V&-n`egZwb1Ahkr7t`c}Dv)K?Lj9D|xmltLTUr?}NZN4bM2+YFrMh1-8C zg}Q#F?t65Gy?$?nLy@1~aesMNw}B9;ZyYj!_-fO%FXm-}|HC3jtSwlnfYo8CAh;-< znQI0xFf*hA2;M^KE8(?Nb~Y}ZGElacS1XtNjLxNTph}QLj86WRotvGXqK(Ggu+Ywp z@*6FR64aTN%{3g^k2|3l&}%`UQqBgq3NJ;Zis`_rUC+tEDCUU02OyN`^;}^iD2MT7LsDah9~S%H zLLzH3Qv*b@I&5p!82Rc)xuX6i8 zSiC;<37M^vEd3CfJ9hS5-%ZFi%0UR>D^>|r$hdegWVQsKdnXtX6KJ(P6DSrM-N0?= z71$J9NVU0^eRS&J2bjR5E{%GU+Xbg(A7WnSh+M01g zTofqm)QYEYu~hu|a|JTMMtF}!YrSa-b)50Hl#>cCj-7<3I1&1eAR@gr ziY&(4w&cKdn~@@3E&?mGF(;^acFuyZ2)85*6 zmA}%t2CZn_=#Fv^n%8>Lft@V0&p6DUE>UznFt1}&hRU`MtTK!q=sUjdTPs=jo~PJ? z?WcN`dE%psS95q=?>kyYe@ig38-A0M3ztf?*Es zF1S%QE4x6mEIP}9WlS4A4ji$Fr?yLzvh*L*)AT8Ep6tPk#RitH{0cRR}u(jgz^_CZe zo~*P7bKnr)uQDm7z^0Y#@ffqDPW?_wt7V(*a=~ir%ysC zdHk9h!Rqk%?9soIUeUi(UeI?6oJJ5!kq}@O+YCs^b9YCOkjptloUsz4VBzeL9^1O( zbq$>J;Dc?b>m+iLn3IiDKz7JAU_z;9I85$p;QIoeA-e zA7IO7sC;lkmHQbrsHO}=xqdxo5i^i=#kG)iU&m}1+kncTHw(NuNA5d}H!HkN3M|eV zy(hd)6*7Bshww{jMjgG4o?Dw%C(b2Sf%SVOj%WNr^~~?49WxZ*K^RcZdm_PTg&^mncQE2Guo= zYC$EI3wJ#X<;h=x;F>937dci6j)#vzH$(*$Sz|jg9>MMu)d8Bn%skB+zm^&XHv+Q8 z=J%hMMAA1t;WOQ(;<5#*67{qYL=h6F;$|aawG-hW3~;h@O`O%KSF&@uZxV(1(Q_w5 zt+bj8w@63;Bq1dzROZWcdqvEs1Yrtt{p~Wp?f>1S4?cmDbIQ+*xKgcc5^i8 zr^}pE@8%!bfsCw(330pS;C`Z_$2Sic-W%Zt=ih#ygL%(Q=WfTB(N3SE6i$;67{4epm3R zI~e*VB5l`tkcKDuTQoTpHy{(73XdK=VJosj5&XASUCYSmy9A`w?sz-mZ>Olh$+1^A zxy-Vw=CF6_j5odJcg2z;)zzx(?B$amE1L5)STW6i(r#M0*+s|!)M=nJqPn$I`Ar## z&g%-LuzUILpZ&F?j8ZRTi(B3GPKf5m>nq_p#XUdJt+=#oJj*E`9h62DczDmALw*G@ z*7XF)%W6~bkeLcEb-?6FC*@cQm(2XSmxW@Bv{c*VbY8U0{N80#J`*t2T0S$DK^4SL z6pw}o*HN)D{#Y}ibd%CYoGpVN#uF9DTx#TwudyKZnQyL* z`p;2S(zS>3*&Y~PJY47K^)GI`^&#gr#j8IE2ng_UxCbfh)TJE~E`v!C;SxGRLd3pz zcQQ}1{2RqzRid>16~26Gy}^;o$1ci#A}joRC?O_SN@ zSv`+9MV6Y&;;~b-N7YK!?nK->Y&}5%NPeuyLs`d7(*aEri5zd`UyknwZ5T^} z*~dciB(~DKNh%P{h4klEHKJ$r0Q%ORHB`6{_d6$X`>}i%+q$IH6yZclZL+?1u|Pd! zhtQ?m?3)C@gx=6^_Dr0AfC$SF$)CuS2nt>Z6Tb9tdUdJLBUL-+ z(nLk3j7yIrAYHLPIxt@>gXMT`Oa%crvcOW6NJCGD9dN zvFoQ4XolUjbkzhgciwwY?KU@oU{?@t_zKZQ-zHT#c5)b@XJ*J)3^ZbjI$+2X#9p>L zb_U9M6=vq_YnNr!rzX;d|8o2@AjfZ>P15`a#}D$A>YALoL@~Xsbp4hVU|D_<&m3Ak z=t~NV%su*(^)Ji62eN!fh%ar*-Z}Z2pgf78A~FF>_y}7*ugFf;z=ubNh<=IUj?N2}YdW30S$-Hj<#wb8>gC6t-=wxp zyd&zjd8#njg6JG27ZyDxKiRag*>6*&D1ybrJ`;ap+V&S*ISh@IqKN<*Vi;O-Iv*TX)l^|P?=;lB@3US21UPQG}wG3 zaZCTE|DgNo)-U3Vy4v+N-c>tncrCB@Cd}S9{;Nw;_*}~O*LYS|8_n<4-SrnK?8`Pv zrw8XhJ9Et+YPag^=+7APpbw7Ay?(7wYQ}x$Xm@<9d#~8~oMRuv6ZCk}3f}}iiE=O8 z;T3ke?vCk%&N7MoWlzu_k&QJvpD&Z<-=waio;+v?KHW`Z;sgDctDt~e^2(Csujd@# z@3vocYnuT#neRN~-CR}gy_M&kLCc>N_5RBvB@(@z_-zmT{q}*i|<3|KY9ve z8-Ax#aGW#=-Op-FR4GA7{-t_h{`s&2NA|a{C8nM?xRHo9x==~x7tcpqt{kB~D9mDZ zr)ctFdpUKWI3&$%+$E$A*)fR8I(BAra^w$gC&h(V+5cdGIeTT01^&!6gR_EUzOQer zc4=Svqi}3=5t86PZk2k;F3XwHo84uUZFYGHWZnd#-t!MjZRR=ND++(uoEAEAV`XP@ zMg#9hZCTrK-cGDoE*{0}yR!*KgBIxYcZBZk6};5FpYrT4)80m(Tv0C%2PJqP&yckP zbAdJ|0u(;Tx((lOg}Ke>5AS(XI2dkWwuAF zko!sZs{hCdpl9A*Dr-h?eNIGq+AlQj57aCjnIfz!-@Iom%DAyGo6eS)DzK}`?8|n~ zmK8B&Flcim@qo+m{5swAFiO02r!IE*)R$Vw5aZD+bD@63XdHD~XszC$EUkxfQIbsJ zLDDi6j!N!Src=52UcT>kB^LeSvllC6%Riy9Rrtl!XOVj@1;U$-OHn@ZS(Et2iAe9f zE@<<+ToSI74`o<%2A|5RK-myS_|0A#CCT=lViJnIDXGde7i_@&MJsC92+96fXbH)| zUcSor1UTww(tYMK3GS9y;WURd^-}JeY2ai*CCX*VUoB;o)h>Hk7U5;kye);Yq}kf) z48)U>ZYIXr+O5pOlghQ!igNU&m^BobRgr5fNps;_Ea;2jdn_cn*wYj#3!!HTqh%y- z#xdW2(-41*4N)O(q3@3%?wG__jw*RGGGNX|!6#Cb@Q$!2&4r?|qF07vvC@>zPT`2d zz}1dO2)AUaisqduILqC~b?2L#aHxvnQV&^(;*b3L{ZL&e{-el$In$l_D6YwOy<61% z4>w5oji?oja^hi8!+e(1?>Nbaq00eq6(7>mt8b;kd@S*ncS^`I3i2*DB0RO{tP$ZQ z;w`yicLk;8%&>~|MFa>`p2{rtDo%yk@_1xan6qKYiEvimWJZ3`6j1EP&nqN4C`6fm zZz*RkQ+%KmQyalo;V3DfGl>~k%lm3WDVK}sVoqK8#fLIw0Y(J48d&i}Db*md5&Ulv zV6&Mmg)WZCt5UzROD3KY#%K=#uRrq?^048C zlNOG#w2FG*7@uAW1ZUR14lWj{*kICheI3(gC}Kg5mc54xsK>V2U?^%8FPMNZqAsoI zG>|51YNQ#kT!$?pfN2%8vso5Y=jLpZZo@2P2tftbP_^vG=Jf?*nns?eEi%k_RW#IZ z&Rem^p0>{(`t>!A#~SqYO^1_TdX)hHM+Eh_(d~VOvwH>*rFv!m>Nwa`nohKW8SQJ8 z#HEYVOoT1XYyLycP$`eWC?Rb~dl_H$&{4w5A41xuyPJZQT&0H~Ewu=4fi3U{BdGkg zu?CE4*BYY@BW<|;+a&|RrA|b3ctlubNbspF(EHf_)OQ(h#{>FDm3H7g{AJo(bNqq1 zyBQ5*MBsSGS^Pn{BwaN|&{A1wv4K}-(_GZ@c$j(+6A=!lTvWL@N%!5J*(11^Zuz^Q zBb~|#pb#|`Ty**YwVi#tplnM&c^Mql1wX61nN!fxSwKw;rLs`u?QmPH&%c1;(CPMj z8&57gy+aKscxU;17IC@KKKUEqkNoUjNrS35f}&ACD(h_z_DD@)pA(_BDuEK=!#WWn zem9;AY;owdLken#f0WPhEO8g<$Ril(gMW`bv^jzj5wP}kT=+$R0VLHz*ABZMZat7L z(~e?)SQ0%aB;GTk4#(hjtLk~p&d`r&hVi&se&$t~{zC9h8o_xPnNkbxjF+&DI*e>N zpQ=_5?$-=~LTHu_j?b$#1xH9C^Mp@QA+jzhhdnN~nP^o`ep?L=OO?FCO@-!@OIIlg zMOZ7eEP)pJ7CI=8(k+h?93IP?@mR>D{d1cb9ie_0epBu~h-Gr>TE3i6FUCtWc&nfJ#|E}EYj-<&d^|n{lVf|+$U$Q zFX*f_l_I3zo0U<21icutxIyorYpVJTgTo#a{$6S%25tNPcC1u|dYI!RS}zSJb@W9a z^jmjH%oEnP@8YTk#fN>Tu_mA|G}7# z`Aq6#km`&K+xinEkhP@U zqBXGx8(0_=X@CBT_lAav#M)zwYQ#=pADmT9Ajt3IXwop_+k|_xu@zfOW90xnla8xQ zxMmXU->Z<%{5e9c#~sdms`XoDS9xBe7>xDU4>gZcG$)?e9I9Q%0=zeL zw#;u3rPw<}%rKK6iLEB~`vRL|-Cm@6g$9{wo0uDc!=9B4&9~?AbP`Mirk>+;GZ)ed z)N&ICPBNYEAyWOMPs$I}3{NB)Rqc1*bWH$n$b#SXN^H^5;SZ+NA7F`HOH8eT6G`iGgz4%Rp(H&#v-| zvr!Vp&jpht(+%lIY&a4|dl>bl2pn=mB8EWmplTkf_hW(itEmQ89rbZr~_z%9#rHk+8P6IuK^IXneVy7k8oD!94j}t?Iwq?5~?Z4xY7^ZUe)9R^jh%Tcdn;iTrQzRkpppC2(wkXIcD%b$e-=U+>x`JG~qYc#ZzMngwTi& z-%Y?jC9IJWl+=VRUavWtLWIp7so;Juq6?g1jz~E$d&t7buEjgd< zCV

b!BT-C@ir1E)pUv#Z6G>GPtY)GFntmk%f7%Wy00e9_@t{%iUIFe)7W{B?aP zfU;^B>B|BD!@nbXyDd^@z#((Il+9RUW0}L0(jiCSMfs%fseM>b*1KGFDS1Rl*B=C} z#sNAJkpDB8CD5fIs(V?oyPBOCbSCfgZ1S^y4@mVMUy6g(pxd|tgn7k8Wn_b_v|`AU zzpE(knA0n$@hndZ$SRXIyfi)liC(Pt-#=nytiabuH&aZ%m#qlK$>K1aceTdDE&j_n zzI@VW(3V}3Ynit&EChvA?*SxWR43qD3d50$fdfW*if1GpdXh3_WJ!S7xNT8~v(o7yAqcg-F*3-c=s0u>QW}V69oz(F7OrL!7_U6+ZE9Fo z&EvNX;j}A*)d6pY%^XZ1)q+~R)Vhyiq`yZ!``n5}vwV4NXBxBN@y6E5_kKRPYW4sP z3@}Jzs|=f88_~G4pZ$%vnvoU1qOv7Y-8(!*Q=Y^Np)D+7u{k?h#5}XQP;ToCtH#&l z7#%(5c1MwyvPTLAA#fB!7^LRjdkJhZjnL!>)3Iu#lfS2lU2Dk^xU5y%P^z29+>x-WL)Civl-?!Pd|;2RGV_;B}!p z-io#VbdkGU-r?H3M=!AAtA|R)Bn^ZHx5U^QvAcD>ym2v}FtOOS+InlNZt9zBXDc@i zQ3lo>_d=XuBBlxpGi=NB*Hl5nC^&fVqKd5g7KI&U3f{2sU+qhE1Bn3 zTV8+nPC{YMYZhV~_LuQreJAv_M@vZBv+8jz>qs_M$l<>))Ixp#LFPmsB8H9bxv2fed#z3UzQeCZl^wit5aKVQ}?Rex@5jNL8P_aQqbL?{4LM2*C z^-fE@Cz1S_3YJnTReXCSyBZTc)C*eF3lIZhw3&)OM7MjA zGk)2O6EaR~Z)qmdgYsYJE+XZvq*`=6W$TZ(Dz33MU#07)$7LJP>4Ooqaw6M0z03&~ zZ-%RyB|E?GCa<5c4vi#;QiZMB>sz zeifmH3+$QKDhy0l?i9>W4GGWRuPw_$~Eo~%!l>ideNfr4=CQQCSI?{JLx*`ejL(E3bcz?_m#(WrIW z$g=gsFKsZ8jKZL=+%YmKg@9+m00H4@6V4#%LjNO=IQ>SV+^~1>F@>8Lb)gs~mV1MZ zPptjf| z)zh8LGp&jwN@jXrM>T$M!0|9BJzkhYq|>`TE1o%kzP9S|1P|8{OVi%w!A>(+E4Fo3 zb@Q0A*Wnr&a;tDKh|ZgyZ@4uHR8n&eYaro|jkyu1KBOlE)U#HKPK-I&88y8@V*sc_ zK>Z1KJJF%|mUh#|wkyRS_v)*T)U4H*8?jHcqCEPO0^q$-r(YivYI|7nPuLa2y*(Eb z0Vs1EJh$BUdXjo);N-qy?8Q}ArfXht$A}f&z%{MyTdLT)R<*v@-98=ylO--Lt+ohu zHoj?jogJamDK56ZpJ{h6sI|n--glP=ta~Esj-?e*k97e8!_V5IXF zA=pEPB&=DfX@+DCU0zEMJ-$JNnkkiDIORp)JS{c%9RG(lptd?{tYCc%G3N?0?>7@W z6h5A{5(bq>3jt+{G7hi*I4j6I-fDNRcTcT{Za&s&Y4XIaDYyCy?ZYNB>gD8>W`*gd zP3P}(P*>?W+qa!LXEEg6*cAS^l4}cZP0}s1<{cHUa4`CaS+~zWaX{$nyyf=8xylIpU!eznn1a7q-aPm3b%Wr#yRThfeIq?k> z<@$F0aWxV!V%tkT$-!du*Eqv)D?J3*+G%6nN$0UHCYR%pX~lAn)wEXq+HBd=GDxG+ z#$4piRANP=MnBJ6@n2_2vAFnAwxGP)w_x@vfu;j?La{JY&iJB&h(GJ5U)B|r+yWRk`WUMgY`{yuhXDL+Z zZGm&Gb8Sz2+(E%n_s#ABHongHy+b|C^4LVBdt|1A5bAuw69s{5^m$b45Dqmi*c0r6 z-AOI8cNqdMRy^A?O|C2oX`8w`l#NmVt{#|5==|_buPY1l<$kIVq$X)(T3fMd_Bm;+ zM`_VV^psO4(Zng*%zp_oyCO=Lv;AWb;f|K(KPCOYVrFU z_de!@!;heCoxUDEZ)yLeZQOXIP05fv8*|q@G^lkQKwtCr`j^(}y0mF3W34Z;9l9pB z9X^gw2{zz9Tm?=}L5GZk*==X)N;kOp5X=+&2j=zAx!l`~KnsDlg_UEbLZr$IfbB=q!@qMFr6qO4ms4GW@45U4MzJ+N7;`9hl6eWKSH0uPsc&m@qx z3j0XmuokW(-9#y#W1AiCxk3uX`u8P4K^N{GMr1)1wUBAc@UiM17&$5mYm_j#oM^a6 zP2?bM|GFd=gOJ4MH zfABHKJfOQ{W8;b(Z!t`1w+Z4Xw~qQJ8m6b;E3nbka_r|Cq(! zK7KqsT~m4Grho$gFlxqh4BvW84Z+$0_FrYi6cf6Ug-8X;_1*k;dU|%wx*?J zbu$4h>V2#BqdlKq?H2!GkIZEnvb$#>g zA!JYF}fZtu{la$f~@`urAuj&;+;hNU%CO$N0>biMZvALoWPxd zp)M%Ni2!nK1}gIcbTnmjf-2mxZCLiOR}$n+hJcVkupzqd^dn{PK51Vq;EQ9n(m3AJ zqYE;kHO2+e6s_ZsS04vBOIicjbUBCU{ecaRLmt)z%bcAcr3}6rUD&SB%w8d-O#jF> z)~=4a^P^S!u2o+gQ4uY#Gi#chMu8w;X^=H{Ndey-RYcDePeyvB!}nE{{}_{SPkXmy zfb6t`@2R6H_;67>r)_*1R@!6=B@?uZ~$pr7z4vDxt+ zjxjaTO`Pxp^moI^tg|ai4figGE#|*QekViof76p~DUNceM zWFj4HFq@fueZS=UdEkRgN{ad)9N@dNqvLMxI~Ao?613N=#>PGClkHSeD{9QXgVK zm9G{?jP-nJ@o&h38h|9jWYsoqr?-6hUIo?Cat>F==G=wK*j$O?gym9TTlj_>L7kd{ znyB5i>2qkBICJm)vfU`L(OC>TjOrDd=PcXWGmn7TnoH5 zreG`G>9heofu*o>a((u9$@@InTEe~oCsR0{Adnk?K6=l9rBIjc z#c+FXcDp#RyHT=ZAZdu6#&!(0o*z~ewNPo*jFkqD&)mS05>b;|F3Ton4Ky*f*Vta$ zrN6WWW(`OKvF1cE{X!Q`g=GU?eEK94RM<8UPjd}6Fk>);0F%-`(5uzV#Ow)OS`igG zuTn_Y-IS1t!hdxQ6q!!<(iZ9t9f>l=Z!6GkR>^8-D`P10l`&@4Tv&OAeR^& z3Mqulf<2qUS{%1(AfrygJf5RUD^{u0s{csCQs&5Jm?{Whny<47AQY)z07J2bz5q89 zU~o5?vcWSO^bz^{&PQjsU0t^o!LuN{YvrY(ap<=KnB|@HB{7nGDP`=`uZD{tI~I{y z+du|%9X+=Pt}?e*0;ZlLXP2Ra{L{XfSMia=&Q@@ zO&H0IJ3Lt#&fu6&n$sPpxM6MJ85XPy?}`IRF0jLRW_Q5@u$v4W1pxVqp$tj^jB|j+ zJ}Y_Q5djtAMbm(1+)1S{msPiN1O(Igw+gQ>7zt(_oz zPB%J>t1#|>X#HlV7-I=wxxJ1l`wqlPBuPyWe><<2n&Z9Vz%2mh1QB?xUb7Co z&__0)C&7E;!CegGY!rxm_MX@;tA8k|p^OWPEk73ZHYIb+wQ!NUE=NbiB^iw0(Kqbk zhqgt}^oCyL=;^>QH@Wv%Gc>!~c{d-YNHylW8>k(suv7ui@0~-Y2(YOcuY^8pMyjh! zO4PR2DdJ%>5m4sQ)NVHaDA!dQFiaZ@9bv5B1fXmAoP(b4Xq1g%B+kjA@>+TqhFpc~ zF8bcKA9cZC#C-zSC#ToQEF%;hCquo z7Doy5q1Fvx60(f(3-fZzHCdzP9;P|!KG11)+cqI{<6Od!EITq;k51OwS3A{3j}TGGo+O_oe4QJ4z@R9Oqf38B&=2up+yY0_>Kgsr{88tm!9W@t9xvJ*)~ zhX%GGsyZ|-UZ?`+Cb(o1B+#mRWHEOWUQo6+Kq4ubXknGuq--*kXeIZ%emThF0A+Bw zbkd15!uSlb7CcUGDck(gIWB>;ndnxRuNCYNpB2{ob?uqc>Nu156_B+jPeq!aLO1+!^of}qtxEVpA z)lK@4=7{tUo{&ijnP5b&SVE16ebMcDfBH03p$ScBc?#>LTO>nsz=cUqr;q630+Y#f zh^9K*;aQtMzAhSw4Xy9}_8)?B?t;@2(}dX$z(N9d553d=+^{f@ipVrlrZG>P?x&j# zR2boACiHnL9e7ydBfZKuZHB@$#_5B^WHaLC`YL>B&iv8>m+s6@)|JNX+Y3YYv4L(0AU(iF`#aDXhC;y#YERAyNMF;%bq z`<9u+M!I+j1Lnt9+WKQG!Epkl!f(DNNm5_Cmu7lfG0=Y3zQ_lEi2!;-s3!=Bb>5%RO(-Ktt#=R%2{WCs(^bKFMF z)rTL717bdjrC4b-#P8(B4FVIm7drr4V6u@1k}~QM~`mZBB~J{viJ}wU)dUD;@xNBzW;8J?|%=8&^g1Hq0|B# zFRio=d=;!q1BIwJI{Y-0xlou?b!bjbbPw%&zxu7PK~9875o0g|NG5ZQN3X z_-1D9h)hcUZJ)9^abM9`wX{ib73*pMTnTncA|kiS_mFFOn@=&;$U{~7qkmIWd9`Qh zlc&^q(^dUop^lmZS3Xwa@NcB6picP^8z9YVh+hpu)yYFXS?)Q!SfhXO@R?4(hw0>s zI%oKQh;$pXw>p*Yisa8>63y|1O(o$jm{KVB|Jpwo*?R2GMIUniDU%H(`#Jba256~0 zOqFcG;9k{3d&n6|Tw{(ox|X0yK{U$`QlT2CBHiBn3T|!EZS^mbZe?6Z<5#~S1Y89i z=98G@yY~BHUl`7r*)4q3hhMF62a0tJH)<-Lss|}F1dL8%7X}#>76l<7nGD|5Luqb1 zot`!C)GFELzQYlx)P`&Ola1dTl~$CXbjEnP&t+x?HeT*mqKp#KeRv#1zr4OZ9rx`% zh%f;t7RYm{1#*JK+Lq4MMG?ky1y6&&! zpMoq@o^;WHh@VWbzR^98B?}GeG;;Ug6lY|F%Lg+@4xuWl>$}@uW%GpFGy;UdeCNDB z5qvvEUQR>YU)bGzP@e0?bPpj6I+$G@Vz9Um{sYp>z12TV4DX!&G~%YI&F*-6r|qcN zb!dPrsM;N?q!Zj=k3swX%M@AOUbCmk@w8YK8FemL=NrM8gW+abhIr*zsHWdLHgj23 zU$3h1`(FVkq4DOmoVRsx487BEz(fVtg2<^yU;gEWWTlg4trr$;ZN|&heE*~c&WSXI z7a%s!Mb^mM63H7{=DH`81}(Jmv+h{Z{q$|&(*17JpLK~hL;X?XYHhATX6D2;@Lt_x zYSLlyNB7A68VZmcIfYLiOVI4O?P3>|41bd%`?a>);_Q@=eFU%Fp`P@5b8VG$ty0yf z(HF;Q8wybw{nqP7Ejo9k^_<{mvb?Vs@kAJfq2Dc*`lgZz7 zZQb8^-nT{c#WB+=0Gw>-Ex_)*7lvN6F>HsK*}}a7-kp{S_J|wr8+i2P9>A*{E|4`C znVcpW8^SGW*;YEo?@W6>DAcvEuQ~{2>3Mpv3`?Fpb@MS@gVj@#y>7}Bm=5b|Ln9mpo{Fr?^!+T;Ww;(baFy6Il2d6r}YxETVpew@Qv4ej+$|VP)_MGr~?_~g&EX=98`jo zdx7qkT|+e-_n(ZVcTU5+g2D%F5z|3-#>ml*jvCi3a!jQFR2rIB2Taya8D2)+&NH*zCL8UnHu8=?bV)kJ)l z4)oWul^j1Z=Ee0mDB#thM~EV&Rvj!}ZK9sh)^=2n)9sg@8{=U#V%f_pRLM^t03!K_6H_bu?lMIlInjjw_ zG5x;7>IYP8PS-`DHOoaikUuM8< zfo|YH1{s@@S`#{wy?b-1ldruQ{+;D5q!J5$4vXp>Lt349QDL{8WoZb=w0Tq!Di7*L zVgPQ}A||V@5h$Q`wNCd^3&RBvR?DWTXE6=78(E>!NE1a^7w0|h81Mg)v{`g_<%)b3 zOI;?~g&BKwZUEyocPnC+d4)FS#j0~ttc#3U>D~JNVjXeINk|Hb9yN)?sYXs1RHw$2 z^ALO_0@b)^pX}PO$3dkl8DKJ%k7gx=0qrZqch@K#PQE~q&+6;ta8O=)W*{t00j6u9 zCTbH1Bt!g=RR)ubz-2)ARfb1(@4(${wlJ23dQh4%B&J?;e6VoSed?rPz65Ml2VR-7 zHQ~3p`HSQqGn;@;%#jqmG(*qT|9orHV>MKRe6Fi^vmNzL)!fQst06=ibvo+jwxX9q z>(WoY>H`5(YbYOgsuLRF?DAeW(fM2d9lCQ7Gul{Kz17dUhzs9)&?Q$r)pE;_ci8xH z%^fb)`x}w@`$Q`O$u;j&In3hB;_B08!ZGTJ;FwHbjv;YYm6GKMA6cwSN(dJ)&{Fb0|K_ocaOqiaCn6t^j#7Q)q$-u3}6 zIv|)>p0%$B#f+ssi1!r8t|c;gt)4G8MV#Do|C7_@cVoLDirl;R{ZB67VA{jRje1QY z<4k&c?d*cn;-aPPb=B7FW#wuZ0a5}(2?QL#q+;Xiy5~2=6;iU(znxtc`Z0?XaSZ(B z#8(1|+t}MtdcH4KO>^U3j*(LpA+~3Q?XlJ2Pm2TI&D1{y>V{|B(}UGLAG9AQ{hQ)* zbA8YCWdY%m-9P;MjC`8o!jdBgTUU}_QJNh@7Tf3GRjW~OQx^bTMrxLZT5BJ6s3hxW z)k5%G+qPA<5fbHFQ;RyLS+ebhacmo(-Q#%cMl73V*;zUl;}@d@t!^VTm4rrrw;3!>>pcWR zR5w$~;d19z>@*=a`I3npzm6rH2A65-VfxtXcdO!J=~3hv*}9Ri>d>n>iNh6` zht7ik5(OjvW@{Vga!1xtHx(r;j0N8|+qT=p;C2w7Mb&l^>+H<839 zYnR|9`q32Iyft{P7^t_TB+5l!p5I&X6}wX2yPpa#X*K3z#ZkGHRypB?fnK`FGmZ$h zlT=|^IzuOs$!GI>_^u?G6^-VMD1Psa#el#5HC5`bairWZn)GqVVo^@cfY<^?ngvCm zC25*^8bT3Y8CKxq!Q~BVmEk9{aY);#>xV6a!g0E=a^W0d2HYAu$7YDU$CTJev`TXi zgQU_O@q8XyGK7>R%k>+#B^yE1Gfu#r5XSM#-jW4F1lC^bCv*uldY%<+3YigxBq{qu z*mD87Y*;{j(;RT}vrQjR#M&VYQ$v{9LK}QoLQ5+wss>7I*xPZRX^h0&c( zY{tobC%@80nVn_XP`_FRY=iFB-70Ch)qF$iekYo=z?49iC+K-r)og2W}yihSvDbAkGkAMz~i|jNu;r?M5C-Zfx-Ew1@~MtMNDy~nuw}H zdkOs3^5am~ok=TFhkfBFG=7C)!#pM>i(xFOBn0ZhetS!nB?4pRiZpt!852#TSL9Q&A4@=0=e^Z0PG>cKPc{|Bd5nfvkDi}?Cf(x0OwF( zi$ueWr`P5dXgE4@4&#F+*$BG))dN&IND6@=a9P2S+DbFb7!^5dxvmmY@%&vT{ool2 z+07gsTB*=nm!fB*c;uj-x$V~OHHp;Ie-UUlI4Wf>7&`R=TD^P2m)OG^wFi(&K0hbyUDi;{A6tlKDC7gw z;dog{B}HWkCZQCeYqD*;*fLO=P0w}6a*FTi^qRAin-_bt^!GgYY4=oG%y*V!ZE-th z=`$*zq~u{tuc}QVx0@yP&`c7_9bd$Rb-ZL7nU*pN%xB{3A}p=+D)#6vi8N|LbeJYY zb(-}^fVUGwrqEa)x0C4ILmP_Rm@3teUD$IckkmBFKuVy4i8M$p$;jo5`V&LeMP+o2 z8i?}+-U?c{iL|$bk>KOUzS0pSpms$7|L!-1qE??gy`f?DBozTmyhauGIsm+^j1MpM zE>Z=;9VW@gAaN6h^D|KsCBk46a}UF}jJ+LM=<&LH9K!|66n8C205YN2CD!fie*+z5 z@3&2nkfEqjI-^Gv+t|^Z$8YPaylQw>-}TMMc7AVV5H)^^y70A|Y%CeT>l|VGMHzDwswtHW-7hmtCa-Q1VyU0YM8c^1 zvGHmAx)iy$VB{G4+m3O*jVa8{%L*P z7Zqds8+B6pp+wHs0;@;ZtS!#k)q)-yq!bwbV{qIFYGsevWc}VoUT;aB*}0MzVAGX{ z50lr2fjdC%QaSYKk@DtT#NF_+aP*(*)nE>JRJAD69`%fF{(Gu)zXp>f^%zKb-qSG-kTCENk)POVp=a_Lbh(!&*NwL{R7gBV$697X$WGgyp=gD7`6~cvF5ehhFma>C?u2~ zT)7pi#90mKC1HhdAPFLpJAU^1xP&~LIpjSI9RgYJ-=i{MxswA6Rl5!-8!vDvn&*s+ zi@-l+2H~n88tn5t&0_zYYXjF9IoJ1fF9ThETD^T-bj;bFWHDu%7KK=@Vl`#NUoFQS z99V(Vh@+WAm8y^!yHyDPOqHUThz>0wiJ1A9^ooc#L`~3Tm7K!Nh&zU=L04b8*DO~5 z4Oj=M*>bGAxu2*=!3l!IZdS$av-w28Mi(3ajf_?oueygYMnX|StmD|5und{&pL2uv zp&^Mgdb7C-Rvp%)lG#!g*hF~+8qpjIN;K>ne-9Rs9r2XO{#W2R(w@dw=5eE)60(#9 z?cfktz0Ap9#}jd@ZCvco!kw*z>G?UmmW0BVE*XADMsM#TmO13y&6)v3 z4{@Pb zot7_PZ-nhzBJrbSiNbC?(;SS}?&&X)gI7TVS9t71VRle3TED|5Lh&G3DJa|P<|7`n ziLHTg9Kn1W*#*Dp;vO1GEe@J<62z@HxP+b@990^k%JKch*45xDP-OGJh$~Z4 z8B^t3c*0)fY#U<1Sze%|9gl46xaS^xv#uK9h@Yji{MOHwxyjA(%)HC7#-@( zIOT{egcr|HhdKM57_GQhPi=U8^0WRYtb3Vfhe_2smT0M2lz3g znAQTPPG(5~v1>cnqT;FmP3|%RJYSG|&|b@Lg^~KFYU1sRXzw*)ylaFiK05h;Hk!AP z#AM55l}jDtyyCHx)RRBHcQxxKs{f-IX+Hk}x=6uAhXsP^l%q?l$)2_>W=W6jQ0z%D z&{eTCLB-8S*~cO&jB7cSX7{J9q$ZSj`BJSX_SEp=fy|bqOT@l+Qx5`h`trAtdTu5n z8ndEN2Vlz%cCqQ*rD|Fx&D z3LsoRA$qdTUrD;4(WsBtdXgtOYLxK=L<;CLv0?d7j#GKPm)1cxBlRHeK~-wmj7L`; zfE^|0vk9ve1@WeDp?k6FZ9opxr}AzI5FhGm86N`Q?|-BtPJ<511n@au;kkgn04N%Y zd7N4#S0O-jsgVyW`W#JLfV@^NJf3kDPoWwi+3%6r?t6+<#t zwzDZ~ww|{#?0NHbqmt8YmswPVo)v@>aYgfCn=_;ZVxdNSSR$AtTrV)U+nOS;EDnTi z*ySDJ2rjS_*>E!x!OI}>_xcz1`b^OI6@EQ{unA7ke1Wijg}fFkr2mJ(;i;XHSJ(v| zWGOQ39bul9a9xz2=dYPxzeMih`HM}m78P{6-+{M-k=0!gaEWoVBBk<>_p%;O#}4O1 zb29G2FFTT6647ZZx!Px?(P0r}nl#k!{-`g@FE!&LHVmI2&#uKRvSXMLhtVWnqUM2u zh?GezeJJnsj?ps6ezJs0(e=Jjdi92JREB>)x|pdVmA6YS{V!v@X-e;9P#6l31ZHGk zy(;p32S3jUi6%c7$FKT3U8c@_2794BQ5rEo3Q4Kjeq&gA9jTKMAumBe=A#kSfszzL z!VkKntuE-~<>jx|=~X!8dNGd)#ldb%=Uvjj1Fr-BE&+sfBgY5LnOz@)t>1NOf z^@Q2h^oaFC$>TUQeAcWlmVbx@chno+3#q$X>ItgX?E|x_sUm24EHiplC~Y%owdnwo zzzgkE00!0%rsKz4PfozUcczfjN##FIZXf>us%?=q6bi>f(xHLv6KWA3)mNl zFk=?X+H`=5`6n`7k0{aGs5x92P85vXUu0h4X}Se?#ogr0X_SD!1X>5bes#zqT#dB= zBV7G?rY@F>x1j^n=5nT&lgaau>u1}1<5`yG5{f-zypUxqk!q&RCv#Z_N1ilUyv)!x zAWaTRhU{3QV^fBdvhIU~(Ei3S0TEk^&v|)#4BJ8VZR7Zcq1#F-cthu+yDf!FPH_Cx zZS&+$E$+tJeBuY*}6EfRsBI~!Fy5Ox!Yg|?0ojLOhYii@h?U1E$;($ z$kVm+zWgJZ0&M})YK0kY_C;NK$F&>O&5}iL>E5b$vGd9sHrGQMsn=r?W!Hn6r}shq zXK*``);;^?=y}_v^QGX#%d%Wphbp$EK&-+a^ z2;@A<4%#N89U>${A#BBJ+$<5+nE+SZvlxOenm?ggyG_7SvbclcrF8C0oGm9fwB4nI zWfAr-f2>F+aU9hEsh8@&2(q-2frip3wmlt3%z8yaxhnfAAflzzbJQn%Xx|7^!ZStS zvIje@RITael84bpXhVgFD3|aCvc`IRO=XoXMl6#nO$1RV0;o zXrJP2MR(TQaB&w)IJKzoHeyvqR4s8_LX}a9n*G;yqh#^d@d3c*u|=D1KWcvl3G?(U z5>{IY(5nTJWkn6S7^#~=3(1BKdlGboDP>{Ys)px~T)LJj&gM1pnZG7a))n@w1~+)s zp2qF)aNv0q`MUK%Z6Fq15LbA2GE*nNbe^73sZfcU*~!VjFJfVd`u)-1NvKgfKd}Vi zFGvk~k-=ZnDY|@{Hp{^IM}=+Lnaz!{^i&(wA&tTiZCvgAb$05;58V zMKV|_RfLx0a|5U%ji-0w_O)R{xfk^2KgZFi_`@=u8W(rp%3o0+@s>YSzFR(DEtvj3!xt5SZUmkUCL$W~4{96sH4oI5-I?{r*&$OO zNtI93njIb<54y>8KG0LoR4g5`b!$%(Q@qxFYv42-9TIbETgm^vT0_kKG|dFsbXDj2 zy{&B}@#@vQdgO36b|;(rSk66{igx&7I`;^kc2I?U+b5ZDfzGYahU1m0eRq(m=<1m7RaVG8Y8oD4#awF|TBH9|GM9x*K&)DNF@RV%H6hfZ0yS@< z0X5@U^rBHZ_Uq`!Y+Uh~%-=%GJ>Ilkb&>14bp9{CzA?J8t!XzMqhs5)ZQHhOCmq`z z+uX5j+cr98$Ii_;@Ar-Sj{D>Om^Id}HCOF5*4%rnSx;3xMLm9TT~kA^P?Ccv+4`_c z43M>`7H-$-c>a4U>mDsKVkbZg{l#PR#{L?VDJ2R~krpPt-qk~W)Si~_T4?8TEbdFj zKSO)ZXx`WA`{&b<_Y-L;B z_$we}YrGSW{Nf^36?_)zbt^S1PJrfl_O<&C_ly#}vE~M+78hZJpJPA z&~G%LvX80&R|)FbWzi`sb&xQu!0Nl89G2mh-l2D4{7Rk4`Mez3&gT+97G2?(AYVXB zyv%^*0%$-^2Xs<{+|*z{HlB zBQFAQE`$YyAuY-*3G)n#L;S%}fU&IhLi|IMpM?I=L;ip*`H%L0KTgnsvYb2uKv@J( zPsQFm>66RB553+s!8A6UwkSNeXjsCp*lT4nTt{6{X@e;4SO|skz$Rio0~!meGU4&> zfAjsSRV6q^;r1%F1-L(;WIQ&=+Tr*&7d#ju?_cu}Xw^B%ng<_ZpGU)D1dk-#EXhzI zN`u7+YU-50&(lE|kN#+?;-Q+49>4)Bw@yHslfgSgDP6kYk*3t7IZo%Ip#Gs#WpG@O zj_NZhsV#NrU8qQ@+lkMPFN2?NxEoaRnoioPUD7D?4#oz3hl~PfTH_Qqk&?yTpzc3r znb#c8NOpz(?V8`9r(Kd&wFd*aVvpISXI)))IGOWyHt|s%oP+uWlrjF}0HiCFo={o! zBbH3)+^Fm7_92s6MbX?1B9WCQO@2~MR`x#BF`H6DVJv&~%-JcQ_KcyPbI+v{$n3`? zb(IJcRX|g{KH)`qVMuMo?&y^p8qV7J@q@+Is{&+)mThhty*FeBw4wSXtwP26G;mXS zu6mx@_wZrI^$_HI-_QdrX3_0N?kj^g(yG84^xz%=IS@|U zEOmlX!DpC${6>QJDmhjK>Mh_V=*991u)P`ETnxbNSzt+^2q(FE3(Fgnjn{U;F10f2 zM4GQ|Ji@F`>ELn8V~fcgGpU3RA3W`EU`yU<*gmoi@ zvKRThn%l0+O|Hvt;>vIo;Z=y>C&-<8}%NW$%9am5vx2u+n{c& zWQ9iP5%`cKR#OqN*&lb8T@#gQ=1^v!cKhCF@)46wT|0ypWEO~sRLAD0qTYDXC``eJ zh`Fi(^`#O&9}6e4$lzVw5dy?aiT?$zF!n zuNipm&<}lWS%M5$iTVRvF<>eCW1(IW>mAHILDXjIt^>5t%a{4B0qd6(nM}E`rS*~# zvp?aVAp%LF=wAd_v93^Ze>8_%7g#Lc$SqxCE1bIuRwTbF&foPqP&oe-L>RO8X4V@mo)jwpD4H6dkN)!7HjNy z;w8H|T4kn8$5XU3x?ePn+N;R95>IdvK@PD&(qwTWCff;^lHUL#xw3$-S#aY)|JlNT ztC2n#Rk2Gc3~Ju(5l~fQRQj`T5BIjr1l#GOOryz`bdc69Kd{Jt%t*iVHlVeQa?1Zm zxB!FgY87b97x*IqtVaIC(<^wJ5%B5ZBOkJ8X{^Ucw9N?BW7Em!WwD`!E3C7)C&!8e zawIz9Lvh!h}j9Qm9HXwD6%q@+v(wpU%Il!j2_ZiL>c#5+hH(r%z$fIaEG zN2aoIFu!n+NHCTp!#$eC+(Qmz3b|Q-yfs87XG_dbLltmG>o>a`%v-H#0(*{)c*sFg z$ZzI&;d_DRb|1OU;542mF$raM9r{#PgyyLF3S*LK!zfTDvEY)FIK*5EOa>{bEBvX_ z@il#lLmp{F8Jt3tVDUOMOs(||#(2+c$jvx+c zyAx#YJ+K|TC9FGpQyV8Y37TC)OUAv^VdNBMV}oK^OritgMPBdt(S?uF_$hfzR3n5T zu(+Q%TqCfC!R;6<#1u{8!^*x0BeG0D%(8R3{w zchdZqm+-I0Ch)OH%l?dfk&hk*F-JGHRgefBBs-;eFPkFSB({c&%73Av@K$B*?%;@V zapp9|36v`Q(HP`1WqN!Zc3?51up?XAFica*B(BUo~Z|b#wCv*AQiY?!fvJZvo z*GT71RrM%?fr5`Cri1}kuqw=p@1@;hs@PMrjIBTp8xP;$>wO5K#U)MXQf1&!hu5ck z^c+$;=91h~M0<}F!Ecm((sDmZ;d&6YcccR_RNTA_uuyU@O*)mQ8+*}dKMFr@;g|q2 zX{T6ZZiN`~lIRI{)6(+9cwZsq(I+E4KcfVz9#Cw+D2n?NC}9KI%|mbHfX+EA0jw}0 z@#KHwp+EfpqH%QVkR-w^LLvy++NcT!^rI~z#0{PrCa42$*DGOGHy}f@BEBV2D|L};70D8@NtoA%S?woNSmC5`B??+3 zV*+9`{0}&qM?9kpq1W}SYHXCPV0o=KXx6|}?Bw`!B|O%NG#PiqL@vQtkQ`*PkTRJh zfs=fMqYxxbqWrF8a(||+t*RjOwW>D2A;;S$j_X`3WOr2UkWQmCW@1HsWi(dgI*E}e zY2SL`vuvs86r(av1%)k&$-e#omFRXZTFTJ5HwtU4tFk%6oUwY3e_Gik)SQHKz=+gx zGWJR8G7_Uq_GXN!wk0 zwv<7_>>54s!(#9p(QToM-Fhjb>bxpg%(k1x=RJuB1^lvGl21?abG@)TM*@B?=pJZW z{Np3G`O@*8@mCC9Rcic5@yR2N)oX zRe=>y)sm2C$r21${*B!u_D;8)$8|7 z4T3Hzto1l8DD1nTk>5S1kfwd%kaqy&pfGAU5$G z{EERgcR)R@TYF?VfCJ7mwh|gCXX~)@bx>EGhn;9K90QF(Hqq`~wrp)RA|D^^s4@Yu z*$id97UUuQn#GYw=%35P1-c&v;gEvjP=bZ+O(9;P#qhONv@0vL4e_%!0wxG{9G1SI zB?ov-il1%;`{D|(2#LDL3Ao7dzMv+Vg9rIgBZVYbxDZ|08kO=CT4RFTlon(l4NqQzv!0wbFSo1sKZHc4(gD65SLFPc^@Z{jDWhk20a=KcJ4}aFQ@pZXy z*K_siY$*Fm*!G5M>Rn=0_pm&UEHfyJv+AISD#4l1nQp8cAnv4uj`qyfz4|iaYLM&Z zc7bo877g-hl)OWVpHgq1Fh!D?Hg})PQpbHwt+~Bk%zo?1RgSaenS~k}G;UO*K?<4Z z3{?s{1m~*r;%^C99KRgkfDTO8+tYZae|YUIuR%;WDqqX3EwbG!jXgK!-ygM4+6&>fMu1#ylq=S3F-qn2nmS{-F1e7~E(iI)y zrY=uwhmC`K->#glv3R_J{$|5RfA0nb)P?fT){X*o>mR$iNuLk2uN!S`cp<(%>>i~< zJ}cDc9lnfXS5yJIqQ6$?cFtAvTD*VZiF$TLVO%vMT+>P4RIC1KA!b|N$@pfT@5n@7 z)9vZ3c2B50LePpcYVWS-8USwZpvmqqoZHMcY-L?A_|Y(1d|mNVPfxN7EmS|&i^f(_7xpyDmWZZ);Q2MLP>cUZPv#AI3s_%z`M ziv40E%ruKTvL-tYjM}EXTFw67z;do@<-wcsgHrs;)yM?LW6U6 zIz%zp{FUf{htJ_}Gb{r%$)FwdllP_e92UF~xn(NH`Rw1b^rg1uVjHs1DIR1&C)2#= znV!x1+iR}%(|jt^YY^ic9eUksx+a+)8f|9y#g@x#GcbQrO`F{v5#xTI_&7Da6{lUw zF>K`MEmOB$Z~;s^q(UzWG)CzhpN}}lRz@Z8D1`DL1BiYxp%_+G9{7Pe&Hhk>fwM)W z_k!ZUQm9e*LEFayMXFXkXTdE3#Ue{60u@XVfIwP1FO(#}?Nci_gr0UlR@f^pvEiYN zyCQ3OGW%Hke_eD}NFqxcX97VWA&?t<`kl)7i&>MFTgldXze4>gvvBqdt`7-wa{C&n z;vfo{$h7JY#)ekc5vvk&Z#+r!M{s@K1_G3!Z;@xwNA&8z~t=jQ7f zFFnr9wx6qNtQ*fK<$PME3Y7{$XC03umg=WXMs`$yxJr}iy(LUkEtALd{ek@T+95{X3{n#sSj;Pc+>C>+9{3+8#u3eUK=l_Bz%AfVg;ugiLX89l z-kL)nb_e5_Qh2E{1)4FLQghXlKG zsnLWz@(OzZuNH{C-35lj(}cw7jQo$KP_O_@3bwuE$NE$leESAGzFnpH&!Jd-PvwS_Hgy9Kre?MwW)rB znH9HFfj^g*sx-$YwB37Srb+ zyB9elmq7-}r>`wu4y-D3S$zDM@_j!k&uS*vyPW4%vhxlv?mG%R=c&zR$AJ%{%K<5v zEprSUl4zp+OOF6d_R?;o;IYF_r~zvMmC^hu0&5hDV)PW++F7j2a2)r^Wu?VqW0yj7 zS?#)f0unjU2Cw46!(*f!s-uX+@WDuf>IyLW1PUG2l#X)UA$wH1S>G@)$i0)nR5Uwf zVn(?)j0kSp;vEa#tQTk!;|icb$^B`GF*+|P(9H>f(+F1e`6NIe8%6U_L_co4Ecq6N z)!C8uC>l;+_RkBwiQ8e6FDhj7-klU$%qk$>$dh{th6`nhkpXZuJt1V+R#^F?l;?gc zj$Fhc87)g}XNvfK^Y1rShCO=$!XQWXEDAZ8SHI&=KdPUGyn2EHb~Pk^cY8rurub6y z)9~QC-vrX5>%HsZ93eCZC5;ZCO~@xTMTPK%&{m6@i<9HCpWom|cf@^Frux*$*N_;# z3p5k&MG73h97d5{_ERzZ{<>!`z%~;+hwmE!A*I#*0ciFFFG;C+B9zensnTQ{_M<`+ zz2m1&(-gP=QggF_+D1FD*2L2~i1x&j77@!ts#f4XVqf%I2#HOku7eo-bJlR;i3`|h z0-YP!IF-Olwv}4J4XR|t%>`_li-Eh}7&mR(-VJOq;H(&f5i6s(GGaL`idPt!%DL)=*jvKa+&q_PpaCsDxjnJlE{CBZAW1HY>@{pO5DKy@|_26QVzst z;%^P|jW(R)RxQUyCweie}r?fFU{j50Gfv zyLtS3{p~PFo@U!SAd`({c#-C*H=8#!+DB-G=BjJDO9q3rW?gW)jb=lj23z>hx~|UA zj<~RsQo?R3=sWbYA*`6!lE}S7;Bnr?{E1wv!FUW(z*5d)(vGgJ-OHa(uw>i2{+O-@ zXlZ6gakFuJH0(n)FXrG=3GVar$axN0Y1xWYUJWdTliuN-1Tdh#lz*;kIt2L;P1;q} znoTgau%Of6`QFd}^0Z!~dG?Y`|Y~BZ~_|1|rYiMpmbu+FAriJjECh^EM zcPi4H%uIA?e=nNBb!i}#Rq>-UWG4mOu951ej>qXVR;cVf8QqsJ)g&aliLd;h%592V8y$BTfGaShBx3WV=(>4F*?R7ftI}% zgvPC5A4Gq`U|$S{eOKiP_-#fp@R&lwqW1!bKqC<-ghnyC6^9-AeV9UW1*;j#C;e;W}=kQG77 zfJQpZ(%G87Ag}fFfTXA?1JcJ!oR~sY0UB!_CDgU{lt$Y|O+JevvdkX^bs@D)*-|kK{oP@C`prc#QBTy$xczoaRaJ^M{`5-V$<;d`sh%8eBQpd zOLsMTzaHY2e_ z@VMpAZ}J!42czfvv`t|h#Vzij@R1Sb6s566TMV&iWHJ?cd@K3w#6E+YA`%LER7b+6 zgdzk;?#>VD*|3)d^2pgv5%b?uj|mp|Q|i!$f;T&fnDHFLNIZ6Vo$=Ftc{Pft6Y#i6 zu)Ve`cZquo=p{1#*)z3RkgA;(>iZFf2D9UvR;jxY?PFFFyP}UsOFbq?-kJH|XiPLZ z&o0ms4HwTGj7qywPI*9YVlecJ@YVwRq6G0jc}taY!Uu_z2lHd6`*#C03`3xq%SnIuUdOXQ^%34PVZt9dj$9Nee*ChrSx+O zw9m7@40Fz4-vGB8(7-jPEi$+G$bwDUw4ey#p?0Zw=f3)<_S7Hke}SrSodXXQE8Te$@0wX_P-nyuUy6ec3~Hp7^~(toM*anifJuJi+@h($l+|>RafU ze_hF4Hez{E=XkrCAr3R_gE|cTCvsc?oUPI~q><1b^IAIK?HH7Uz{_^Y*44QsxUPlX5(|cnu%6FdT_NR?1o{wE=pF zkgF#WwHG6~PXDK@(yR3k?W7-b36N0iVWq3Vc!C8Xq@&E0bd_hw-+5Uze2DTfc`W(! zN`2u|W}Z@q`xuZ|xQ7XjjyQ>&lCfafKN;dmR1zePvTzQf;S7iXZmu~)xf?;0R?oAj zSEwhFahHlH!z9H{^~Xsa2C9UqH8s@})WF=u>_gvhVk5s5t0h;cw^()o+p_S=Qd9UZ z7UlXl%2v(PgVRci;573yqZ&}o?ElHy8j2LY|1azz^-I&&Q}nUcCj5F?jan58HnKR7 zm}$s)y!FEelm2IRvR2>*l^(Y=ht%qcM`#GiudPEBWxQP$JQc_=yD6{c1WwLF*9f$B zSNT#@NH$D=0;DTpD11Y{G1q=nYNxBzi^c9Ga-?mIk7#HCaZ4NMxr99vNJlv5pFXmd zIEg4R$UKN_c(KU3hhuKCy6p!2TCls!@e^q$Dxp5;+IGGYf&B8$Qg4GS~5y+jKv!=QFRXC z%AWOmj--*XecKFDDq2|k~Xqzw-~lS&9U0S0jIC~9sS87b56@jei7aN}0sZ;;5`jT=t3na7 zCskpJS<7PCypq%+pF7ppYY`NxG+1aj4h|J%H^M>~c0N@pJHYP$W0~;xl}1jxKpqnJ znNd{Ci`{G99&{NnzBO#QoVWQTiN{isGRSJ*E}U9jvWTtnc5vkFPZgOqDfyil|bL>WWvP9OMh0njbc|1$BSZ z#2VYN7^8L-_DkMrwI?~is5cWF&s;2>MNA`Dt)@b%vS^u7prO@7L(QI32;zm?3){gE z)|uEut}#@EhwC9PRXeHr5CqB~vA%OVW~DP32`lrPU&Z2;{kJZe`PA5D(3nAVaksoYF|x3=-Joujo(~jOIZGHzB8OaS zo)XedzcRf(EsG5fV5XrUoe23{YbqngX3H^%X)S@ga5<*52;m2|7FH7(J3~%M?d$6! zU(m@5lhwl=PWJ-$fk2FYstB~j0Kb^!!t08*5%~ZtO*L%W-9-?)DoP@k@tQCIY&Jz? z9uo{$+`meyTgFk+xJHULl}ndfm{uyNo|oC96~@VmZL-nxT|P?$=4e?(_$D891){+D zV)v6l)~dl%buM==00 zB~JOW8HbRlaL-U%3Q82-SrQ=>GY(Bk?r4>Qm2-letVhWR+?q1Cq>CV6#Ia*xFGp{X zcjCArwy{~bCS{Dt#003iO<4hn<{19-%Z(&laTjjTuq|7eD}8-yP{+AsNx4>#E1ZvS zur!62A}-e%hh(q82y=HZZGM^ok!u18OVJXJxV+*?uNlNu2{HK%_-eu5G)&kfa=udl z#5TLM@?-;u$}rA}&*OdyihwhrGaAdK)?NxEe2$TK!U{A6jdpmnvU37dQgd`XC}L zd|`s@GhPJkRVDRnwK(B+ACZA4%MD^b&NK-`S{`B%=hOeXDG`gX*GLhHFEJ6Ga0B9w zuxkBAwEb?x9FqAE}@}PoDdsdXKUseUH z?A~*9*RK>;=kQ8&!wx(0FrCQc_niIP{rj%ETu6MFuMrZx|a}?@SSynxv<0dfU%bLJz zDm+S0kXUVu zQnxuuFvCEyuvX56d4=l`5+>Jelp_y!Z^E=d#|Sgg(i2sVzCHZ#NdQ2tsvQj8kxF!= z{kiUH;;B=$M5%`KRcU& zfAGKk?4xUuTX?hBqwi-NirN^Kfvtx#onahjDT z$5yyFst(c<$Z1MyY;hY@_TpqUy6bW7M-QKZqBrd$f=T8x=x#NEKs(pxko&_Xs|Qm9SX=5ep;E+2kg)0HZ)$r+a4 z@`@zJ9vwc4w$)4pqkqB}E_tPCOp$|mShCJPY*I63trSunT7n@drx;vTa?W=Zop)W- zwJ0P$empIL*$nxFc{%Id?NwbgfUmptQ8lnQaAl7qqH^hP(4`)l9Z`vHaHM5m1Uwuo6m~!u*R*v5l#?c z50xgEwyq#(T#&c}Ym3Cv70@OLnabdnY5qvC8a<|$Er`c)-dlkSzSPD-?v}^wak`T) zou@OL{aHeEv`s)W3+vEH$>k3ZZ~u<+HRNQ;iZ!vvhb%fbH9L4K)m)XJUdjei(*g9R zHQ?>9EC~<8JAlyV8}tjg!A1WpUX#b@$1~<~lq%hnzE*lg%W@I$ch?z|@*iS@Fpo`3 zQrH0F$!CdUkd#WC+we7k@LLLM8KpJpZ$fkAzStB)QPFN&S9XFzO)!>%-_tS(@_1Mp z3`h#I^^XOzW_6-msq4A#nHo2noxmJ*+ND>LCx;e+u~xuG%Y z5rs=+z)99t@n8~kIA=5W#!wx8sCspO3Y0N5p+@rs9_d*>H%b@bhEjCx5$u-K?9YHB z^`-mUN4?K`o~d{`a4k}peQ)ZQGEv1fq+FCdlvc;H;ec{>ZU}(@i(Vyvog|c?P#2%S zqZ$E^mHEEuec!b(hNRfKYnHVxpwi<1BRFU^xH7bBUG85PG`O_B z57N+RaQo)}j?}p%?T=KOD636Q{6H4Ln|GzuLpom$d^Qy&9EMUsX$d@fIzL zr8oZ8QqQxe`EH1vL^p48``jo0u5m1(uf5`<{OT$qW&eeNtAToyU&W~5nI^?Wu|vTl zAvj`nC;4I`jaQnz~s4B2tDN#u%iUUdz6GunwH;xhT>rVvXfTXi=BDeEekT@kGb5( zPm19$m@FQM@Jrz3`)+Uh%q$Co_VJ{7KaIfodOGA@g*_c_C(}O#fp=W+%pMkc*m>7; z^M)t$H*NbXJ;7O=#_Z#f{XR?_cH#xm050%G5NV0=PY#1rgF2bU-+F+PS_?Gu+~}i# zM-Q)GB1j5@6RzO(X z^Qyz+ru|hA#N7Bf`b^P>ky~bzFTN|fQ~3H9gpSXxc1ZkZqXz$|(Et`j7hN|j-Y9rkYZvzZ(zYU zD+DrCX7D&!e*GAhvS@z2q&Un=x0TlD)6%?9cg^o>>OXI{mV`SiYFzLy-3p^mste!h zP_$d0kYD=Pop@B4NtcN;8~6#2?9C$yjr=NXr#3x_laAl!hzEavo748Dkzi6`doj)* zS6CxaVS`GRrTYFi1{N0rz@#A>4Ne`jYF$`b3{Fq(3~OB)n+;AG{ylvVXM75oEGuM+ zyx>bNmHNNerf8ShC~XLtCWg-N>6s=K(WdR{d;JlSz${! zp*32MR$m!HW8K9Cr@8#A2TF5UD5dGQX;s^|&J^a~$t8~?G9UO>S&Pl`EbRJaj<=At zI%SUan|L4v-xIXf$s@1+WAfYj`cw1w;{DG`ZC4!R`8<7)S?mNo@>=Xb519XAIvjpC4CttKA+tArA4Mp;DA z(!NIEyRa4M6G?H+gg62a4vlNV37l`%XBN2_;NFp;PUj*R3e&q{UP_^)vzu0qtplRz z!$g!O?31XSP{1e)*^h)f!^nT+=|v+k7=bc!D2buuRtYXZ99Z~Wa>k>l8|8Kxq2)7t zTGPr2Iso65)9(M90P)=t7Wc}G*V5lXO>qyq8|ZuQH4VAyCZ;hf@!;YdY(wZ?wI*eVz|2{d5_Xyl-o$9$KqzGIJU6r$3)vq@GPf$!Mv4ngAs=0> z)@Chb@ZlBl`1oGBJ!aXdY`#MCGl`zivefIWJ^Ea+VBVvWZa#}dl;$@;*o4SMjHA74 zfCo+C30#`104h1^ab2(+GsCh!v@(<&6{$3muu0W%5!$*0Nc*Gb2+Iai&YWGLOEjNP zCRxXIM3cR$$M@^(L)ryvq{Ks&Zj?h_rY$YMK!G2z`p*Edi}b=Xh1l``M$@Rs-9(f zBIe35Zi$5aIgitZZCj!8l-WQ+_=|y%G7|HLv#AtH19=ppd|B2}q2Zv}^e#l89E&PL zZ5#lB6LDnb>$RoF<`B3}rh>_9H-uXDeCfgfW(k*}Ro<~f31~UZGaqc=%xltQP(#R} z;iD_F#d^io^RM%{#4((dI?H%scbR%XG9d@JEO%v%P@A|#%7#RT_t46$=Z5OiG|Dsq zRyfHmH}g&84vJQM8LUH4?aI ztzP4>FN%e%4@@K3Y=(d}aMjDArEF@j2DE(UXt@46va_g`UVB?eV~9$2Q#{l?Mm(hU zLMWTc33gmVw86H^re$P7umRuiKYDO;A#{IoU_{4L28Z|1V~9?u6C8oj6qqHj$9u=3 zzguqO7dpbpJ!O%AMuGRd#U^%<#M#wXqTF(bL*P&?cnGhhN+{KDwJd zIT_U)Hp6_jZvYCyHsslJqrGm%se_(wb#=nb#o$NT3&Rew=hoku*Ad}k_=xah@VMz? zUUk{{bFf}+mPp+eDw{^6w2n4XyTrC>*lr)$t{<769@BJJ^wp@bUc*_by)v-=;e;4G z(!y-aF#`wt4{bt^`*=ga++U4xex9|zK=^xjazyAFa<(SHe}$T^!eZ?!2{o&XwDg&d z98VT4ejQE)C#ba77hwH{5?Df~F^l{ocC&BLVHQWA243gX1h(JZNf6Nbr5NswKpmuk zIXHgk4sroG({Kj`l zf&9id=0889VQHV4)x6dMt~(zgy^gPWzY50~tFj{vJYj08E)g4+U)bo6fQIXDowZ|O z`3e$@yPqG=PFLM9^udkm}#-W zXecJlwwDZR0&E_+w(9UNJ+LVqi z+AXa7(ETB@=_1O6P-?mLoH1kH<&BphS~I<;Ke)V05f5dRw-4O-=*TQVKvrUjH|)&m z)?{qsIYUdm2C#T4dFS^UdxcqInhuNwxo>VI8A}T$)l_BKK=LL;<}oX#f=%k(we%)P zg!#)$?m-p$Ynxq-F=%P-G_HBxMfXHMhtL`3xkF3{993Q{Nh#&y6}#G$Lw0f1_1+Su zhLk5*bA_D_qM-!E*MD%M7fqs_9RV$wi9FJ0e)aewETY}I&S_&aS#$dmqtREN)ck?X zp2KPbyh->R05jkO31IGl1W{;2x=d*c=CPBU)HHS9T%%@|A0lM5V>7StlnTs*>UU%0 zoZP$osU8#jz)dWYIcAq!g(Rm4*b@{QVrt19G7ar;vBpmOso3kYV-DRX1u`)BXC>$S zn?VS7`IJPG%S^`>FJsS}Yr_1<+o4%Lp}0GaCBT_4nwnZWD7rQ&_&Ye$_w`)>cXKzk zYyVxHe|M8QP_tL#$H(!!*V5AC^R)3`U*p%oQKXOW`!Thbixn$IORTh-zG4IlojIwL z;QaK|_!FCA0p4(~N^I8J%|xj?f^wPg2kEOOsi$nRhW>%Y8OHBLje}O1T@TVa6{1;G zMssW>K)Ik1i@Bs-C|)6UmrmsQyZ4Lw!v%g5BE(Ad)_4-KW0;?eiHpwm^#O1RhhC^X z`aC)=ONpV)p;V^x0IF`4x5DHK8z;8x(v=X2RDApLDQLN8!>Y%9i^)UwZxvqRy1P2b zgG!DfMNN8euF!3j(o>}@E7+6hhBwhZjXDf)Qq0H_g;o6zE+S=~(&%%R5K=>Gv8D0f z?Gptn4kAXhZ|7}KVG$IwTOsZASt>Pnc`86QN0ss>lS+X{C@b~ZO(>Ok#QGbc##W43 z*;#ui^4~q7!=8=xsAGHr?8b@p)TNJMykQ}-NPdn4vv-n~a+pf_#z_Xh$X@2{%s?HZ z#0E2)l#>8Sa(?4ihe{eL$|5u%*$cYPp`Vx~U^tM*Oc8o6!(AbcSV$L;C*k|s%%_WN z?BrB6+~?ngU&L}ZwEaj<20}Q?xXH>X(iB1{OS6??G74?YqWh)mTmR2sPOv3k!e!|D(x6;9dj3vspSM>8JN&j8hjI%=!v3T}AVC*7 zPU$5Z0%ATfXOPCam3kIL@<38&B>Xn6`S7Hnta_bwoG-S0I(b&FsIU{%iKX4IdvXj4 za!0(TvIv*IU_POrCV*za$4G1Gip?mle> z+>L;Y!&9dNui(&7B#FAyDY>P2+dPG5JQ z7k%{WnH^n-pGXW6#b7^`>|VwD2}rW-BYC$mELYkLhtx8Z(n8L?E6}vqueYu==EiUI zOYd?Wnw*rhxDC$NwvE7nRXmn?5&4B%uo?u)h$ZcP$nznMBR;r}ewOzvc7t z<$R)Q^;$rE(U>Yb<&saM2Sh~@nMp25hcH1jexm1dmo8U>wAF>ys-0pntw|u2LLN8d z^ko0XN!+Ogwn|xH92VB0d=!^KjVK_4LP=a-)obZ*{g@$inq!SImXF@E+%VgwZaj|K$Ccvz@(ypi!s4d@TB^zBN|hrk^RN8W3%8C7b%ePN zjQtWp{xZJ@Dt>gE#RXXGDqSyoU3Z)NUb?jt!>#%`vFbI8u&cEb;tq?#k*hls?U|_4 z>*|5+24j)=}5<5AcgCj85DJW8fEI}yDXg!Ts zE&*mDMx@PfORk*dpU6L>#7|0|6;pG+Jio5_J3njuF20-&_(1xrYzd}e@CHEj6OxAh z;6?*|Qc~HW9R&CcwEH|Sdb19J=O-{>?fRE*QsR9Z=S50gnI);(_pSYd7)TP|5Y27@k1vIrU z@Ao&`*)I&*eP+j2z0vW10w{hcIR4q;gLJCvLS=2OO!`jqj9EPvdUHJSni+bVa~CamSCb*-AC#k}vAFDR#w7CNO4DJ%ME8 zcSg>Fx-I+Ve6!>CY3uO(&=>I6`t>^h(Ut4{^@jZ|jvpTn59j@TVB@-)Uhhi?-tUcb z%Dm-O|C`=IM(=AM>d(#1xhg)OT}{xMB^hKG!s}E&@T~y?p+n$&GgUfjRk~93cL6TW z1E0$v+{xeHh?j<4w1?qkNmD^ctvTyp2EzEsVy4`Ce=R-`JW=jh1mzbRIRxeJ#FrOT zjFx#)St%^pE{DnGnt0%f}!LLL1iUCAjPxa2{NNYmEv` zD2z384Au;wFZU1)C_FTOb-75)U*vUxON-gMWRQfHWGGPs#qcJ1KFZPxRpNpkiNHRKJ)=dh ze%C%~9Uoh5$U~d8nQ_^Fp1atE<3>7P-WIeK`qsK17Us(pFv@UMWA!$poKH|NJn zZHgXv^De-NB+KgF;E92qTotVI-5Z)<@)a*zax^<^@`^c`T3H9+~8Z z@4F$!d0eg96~fK&lyM2u924SE>=^nKQ-}5Dmw)-BMYu56C+YTqorA}xCYa~)? zhbEtAzIzyX6n=>v%Lc~M-;%T$K*eCOzIShUz-6W$N(M3kt#*c8E9WVcj^XNse`ldh zLh%b7AGpitS*$g@Xs<-6g9x}p>yc?75ylT9&mwB9Ku`dm057+`$FiYHiTDIxOvxV{ z&j$6MRh2e4(<5_{%@JA2)t%tBW1aS15H*B256Z3+C#HhApe;&v5T-kqNBywWm5WM^ zf3kY3HN-@|mD+D6g8A?$hk-#56!1%+K;p=ZPcs5|5c8z`?k_1Qh~x-}SS4Xm&Kic= zvxv5w-!rT&xJn3~v4C!;ybSyCO8-zpVT1!bLJD#$cvp7!2zbPk$ZcquxT3H~sy$K5 z<4I;=d>L7#z43ywq}2{*YeyT>hZuswD$5!AdwfXzoUaObYrHzf8`0hxR$g+EBO)h7 zCqV#)4i4P)*S2)T7r(<=JlgVrfHx-69E&H5anVCmvFcc;(CGb;*t!OC}or4?`=^&4JV&+VA1EmeJubnb_J$jH0TLavx`6Lu)M7kFLExB< z6uJ<^MT>B^$ROS%uZu2A<7vbFL} zyk8<23ZdRs3+=MY$nj3Eb2FS~v!6Ns^quMI%2^R)yp|F4)m7^G$-nR9+QqBGPL;RNX0MinOlAYVl{o(MifC(D*=D)gHifI(sSv$+M;Q%(f zrKB7>!5G*)HF(*gRb4Non?Sn*r;o$#@7zdhra=c8Ef-AkBbxvYm&zY** zlspbq7#J9zhlt56HyhrIhI*D@TfZ16LKE}W5TgFdO>La&z5!GzFQm87lTkLaH&=(^ zzxjDv{A7419P(K@HEE{}At$zm8fE7_u8)WQnamX*uePL{t9v51Xx8QJr#sAe%2R1y z$#v#kIH|mUM?2Pj+CFt}KwF|3M0vu}$ckJ%320T;5M=FG%my8r0}iU6c8E5XL7rVW z0?4gE>bWYaS&-`byojd?KkaB&=oMC;Qf@4c7xEcW2k-(cL&FTX+~{h_VK0ppOlnq` zKj7nldO6_c1Q;JxiKk z{hyuW*8hqQL#6j+#f?+2v)Uw_;YUO_uCEEEQYH>Hb`+vbh=y~ZDFeSmj`Ai8fpi=E z^=;#VzLV#Og&c=0l~Ii1NoMSjo(q?;pt?A7bG2@PykJv7(IgE!sZVel6EPC2>|8c0 z7w0VjF9L5S4I_dbGSkj=T;o}Igq^Q&_+84dn?~Z1-&H=s0(J<69yTxqt5p?bu=!H$ zSXw5KwQI31Di-8wGZ3F|fD5x+XMlU|%dIh@27z5^+zfV}_VhE*wXy;1T%$JjU*Xzp zfNO2n$F)57tp|Lo%>TXUUtw+gt>5&`U9Ou}uMB`HAG60Ybp0_5jD<;^tX)xmQbwv( zZsj-!CL<$)z&cW0>uXDVR^HOqFUA;o5PkZUJPbV|HCKLjBUq$+6mgkO_G_-aRI-3R z@Flz?JkC7yS%WNGHys~u@O~Gtbs>oG;Yw$H{+&|WT6d@~&xxp<(zsp6_jFDxn{VMY zsFYugn2qiWQeYsKl02EotTi!y?w*b(`HI@TnQj5qzMXR3%+;%tNw_V=rF8_QOyyqLeG)|V}5Vw`*mx@U6;z`$^;bPHC${( zaZ$D?HNeN(Rg3#PRLI9;Pu_bOLd-i=-^JMfzGI4?&~`iT%aL>`cA{pVxH&kru{#8% zy)z9|W3KmP!slkFrn_6bF`K@-@7K=wuVl6HFnfN>w6&9ITZ64?ngGW!*jPHljS>| z85f&SOqqg0UMCTVuD;#c!$G8LEK zE253Q*b?=Y>VD|$7Ph95h=Mz}bTd{k_>^`5PwyN0{_i{j^wIGND?bA9K#)bOw9YOi z!P7CDT(u=Zjr8p@IyW`LHBTsS_^Hn_S_WfkRM(%r$t9(L+Ga`@tMszTjnYheqHG7K zD$KVPL8j}iSFd%QpRKML`{S}~hME^=a&Jh)chpgY&P}lP>X<60NUBm*CnY&vgW5L| zf(2p$>r}FFF42LMr{4E1CB6gaP}OnPNld|s3DZ0j?~X?nb%qMM1MrkcVuNmBUn)5x zADPuQWRnn}#zm#VrIa8JGwzH+GoMl}(u)jZo9bRK$TIH^H9^b1(o$?-0a6C)a%Rj| zj0lun3PSEdCQelGSMi|(SK1q0jagfvB$-UyG^gviNrj3EBtmv=7Z>#i!l3oJ>Z2*c zGer4^^gk7aHSaB(ASo%@`+_-X2U=w`xWy)ce+U^U6tjd}@i)@bMikye8`@~9hNfmH z?XG2GCf7w>;+cl}z^UacbOJ`~Go?|Qx*qE<6|8$W#E6L;&jn5ILj*{Mi$rp??>yRT zw-H##G)}VfAyaYN%0uDjFE{@#t6=2p{73z%hGcuSWj4vPfO){_O<-VlJ6TLQ&9<1C z2w{@jWEx7?Lg4xM%@2J+kvzm|ar50SUt6u;v4z&kCJ6he%3U3QC)@AgUG^cfcVvDk zCQ2R=%m>)RB=Qu*FuCk;x1(VY7llFFuBDA#o*W*`^`=cVNd|p-6%IuPp}e_aD5$_a z#L=!G!+UR|w78^wuG@FF6h~njWjUqj79e>>^-+dAtD$`GC&Vo zZ$1<9MK1Ji7jM9E3YA&&wM1kRPBUhX6 z_qi|xc}H@)aR438<~tUcxAn5}MR&Ua4TboZk=}g-bh}}CTTU!G!@4W++YKi8|r&_3!Zx9=$ ztgq|vIk78yFP$%(xYsp{+Ux3cp5NFR$(n9nh{!UVv9djD@8q_1?*xDAGbqbn7uxPy z^mpr@?)?kO{PfaXC`&$;;)@RlZ+y;FW| zvcKRRuA*PyI2;K3Pv~>kQA*(duKQp7=Ku5gg;?|gpO+^IITHP*Xah-Ev^+RpbTR|& z2Ye!#URw=rxjn;Mx>M+L3_JBEMc+7i)7eH^PZ@kQ4Y>o|wV!=mjL9Q4rPQ_{I+;AV zMYByIubn7G&ML>mU|$5gMSkP@g5RiqXJ2l=CKb@J>dT>~i7gGQKBxY`_*Xj2Vn9DM z4dqh!+VGl1EGI5awopaw%Jeq9o2B{?n3J>Y1MO8~RSgHF37^9i65H#spGz^&vP-aG)|U&|0%1&< zi!*>)jq#>v065?RnyoqRar7E$U>*3u(A!djB?>d!L` zc||7@4QY6BBK?w^ZvCs&wZ`!^+{K>#$5YeCbK6_07a%VBKG&{FX{6ky`G?2HQR=!*ICz{8sN8Ni1HfcM!C6AkJ9x%4AH_|MnrE?Ag8KFLDGWUc!rB|qtYG4oIE4Qy z$E~#o)#4ZzWm|BMY)Sv)z)$m4__G!*MS4uh2hw#9FP#|Ep@TpfR8s4w_xhX$q-ec5 zb%Enr;w56P2~;w?p9n*P3<-2vk7HWA?&yU7tJe3&C((Ube0zZw=_9vP8a;h~E<_b0 zAVBB!u91;KzG8e2rVQ+?OEu*81iY-369W$wmjN-TG!kjE=Gel-p6sK#q(f zNVgIU=7B~gJg+>e+IF7~O*dVj!g?Jcl8CsyhhtrEV%g&r6zR3KkR11dNBj$69W>^s zbC2qL;bw!0SD3ozEj2vZ0#ye%$3aiRtnu7~C*rA>Eg#&ni0nomCR+CjZ_oI6g9Qa- z^c9KL=6)f$yx@Glx93M>H$t|4wzUx#i|N%C_RzmMbJ_?4%cD!7!twrF?DD0Q`tvoREF76e^(QPFV<%|7nSE0LmF^ zW3?`V{nm?TlL{%a4RcXyRsTw)h-DgN3e{%7nZ^LpG&5>BI%1w!c~{SC9`+&{M?|>D zqM7bC>na1KrTa>ZZ-Xx<2^@mROW{Z#MZSdR)pR*35_~eNFNqYqEo9#6eYtfUpA+Kv z&?o@*_ADJY2@9Z@m!ad=eqUY;o>#S<-fmo{$RKH(wHk;Q3RP9Bl`x3GWMBkdKm{T! zq;I#>MvpLC)j`;i&2vo3y+z+c&y28Gs`fu|R!AGv$d6XMbquj@pnIsMUG5=6;Z;D% zbWJJTbItj|Ne^1^OK@*N>+@O+5M>KEW!+p0Sf2@}?Q`!Stkeb91YyxdvG5bTG+A3HPXg4&7*STz_t@oMJ7i;V5~#as!;Op~VFbf?Ehp)J4xo zW!_mMy_CRP5MG6%xAQcA@`&1ATG`gFG-IKw%h`sDa&3uNHKz9u+!B3WF}gvORiVMf z;uxhqG^zI*@;S(*bImchgyhv2I4O<3`%57t-vbXM#9!(_mjc5sKuMhzU2Hr;tiZ=( zk^Bd!wPJvN05eo;1~*tJ-C-#Gso-<}y)NwWc@Y2;tWR}KblpIXAyO--{mXDX%nWoJ=*(<-C+ttO} zcA`WQ38nRz_>_iHB(-KZSLei%RoG@7G}CpVB5FXf{k^0L?f|4hT}e_f;RK>T@f=2A z2NGO&igP65)#Gedeof)umUe2{Hy|}`mnwh=qhtn4H~*cG4UBvOTM!3h7;5LdU}i0@ zlSqi}ZB(K9wApbWjkO#MClxHQ@rvE^0mx{dx0!~VYxF!!ls3{OK2LWHG(MVwz$T$y zRZBe`C4u8*IHiNj`N5}~p8?nQhQlYHELvLhxP8=>`X(384@+la z&XzVGyAtg{nFFC`Zf@jn^Dymmef{UVt_SsiNhjpFc&WbupPp5m-90Qk{Fw~!FJhja z)x?iz^ayew)WzM+a{U%9X&3k5r>nNFs@)#%DY@#g z3yQ#}&>CRYZ5D+XRM{gotxO2R2}!lx8?l-MI5W5!xMSHaBp-TQj`mCFRw!=VfFM$L z;Upun`rz=c)VN-_J$!J})n#u-iI&``bYvw0Je8bJY&bgD)MA|E@iF*|+5h0Ex|qmC zQ>&bT4qiM?j!HU^NFvv0HXGNUv_4!Y#{QD6TuinviuVOO8duy`#vFhDaF77R_cQ4- z@Oor8K$o8sjm8P$2W_uF)(qs=m#?YgCq1D}6qK_PP;!KsYRJVQkVn8>v%maPn>6}x zFtaUt?n@o9!gX8TT+=q+TfXh~_P^#TJ#Z=bO5omNb@lER+dEF#425my7I)}@_2?2` zo~7%yLA}|YZjA6ijXhQ3C^CC2Z&Ai`SFI%>N=TDQxC-yl9 zA7Y{cC99jTm|nFaUkO^H_Wa35cC?B85|I6eFwa8yX(Y%cTf&fuStprao(_5yRsm|t zgQ`|mQN>?@yNxBQxm|N(gEV;zelgmS+3MHBIzM&xtmJ8(N8-%dlQRvfkR2Pab3N&F)1`4s?aD1d7N zY(>IMI(ho62P*`+1#22$;^}=9S2F>D;ywyvkhmq2j^MI^8dc8Odb|Hqw;jCAYq#uoPSD zrNf}|S*7Jw;4rl!HQTL&H&m8Y$51ln{ir0g4V2L^S^Fdfn#-Pa%jk+;wS8MniJJG{CEEX4>j7pCP?B*U?nuxe07Lyl0ceuUKUHyK5A9NB)H2xXL094kD@ zz{6ny2^!QcX^*fe>oV+%*^qZ5PI`+{?lsFa{g7OAq9WBr99z+Zyn#? zQ$k-Ic4!n!dDa#M-`TSQzlXJ}zk`B*t@DTQS4Sw?#Vl*-`$t()Ibr8SRH`e56c-fq;1~UM zy2bpQhy5{&I(sX?eSxTn`Y^&(M1Ru!-y@ zTYX9<|12EZ*dAlWbXiDHd6)%d0O9EI>(UJ7OgpidTMcRDQICb2JZ!LoPJZwYw^Xs) zeJwzgg=mO@Uey7Z421ItNCKvCnciV=vISi#B=HgkR*jh(=)hn*4413_yclC9Q8lVG z89F zx{jrAVKB4d7F2@-{*-d^GP<4lhOO)nfv~xlF^3jo0zTErMg<40Zf7}tL@@ZdJDB+S z8~n`0H;N20+7aErcjqnq63f4K-U<-&G@~Tz&SuB1y#UazmiAF1vkop?j|@DYG@#bk z)Zaq9tM1|Fofo5&PD+fE$-{ucEy^WgpLCnK{Onr0fGsuz!F)C4TR>__rEB57cZL@xewxP@I_9D z^ZC^zCI$DPnnI7MNin@%-)+6|ZySv=@o&*AEcZ$A!4V}P9P!~NABy8HQ~;1YB@RXO zdOK6x^YAwhhEsO%LB?5RPV1khw{jslR6ejLH^tBqbvMFl|KMEnZIfaC(K2NmLq11) z=oT}dWZItGKF*s5VMC-UvOk)!`oLU2NGH&BZexB%67khQomyv-IAd+=jI%pc>_#b< z6j#>r7Nq*sIXMPRdq-AVb-2Lk;RE4bnb6btuCYKPf=aY z6GHX=_CwIWFGQ#q)6WyesLCbS2eh)*G>}Rvo)$nZe7{^F`pYe1`(QzHLo1}G`vv^5 zgE5^r46;ZCb!`LBv6M^fdtE`-g?wjcnpk!fy#4Plt{Szz$E4%{t4YHYGUj&t+82bj z<(A0$g7aui-XN+Pk%CgDsi;J6p?mw58ZW{~y*kLhRQHv4OmMryAJz{l{(lCud%cY5 zUd&JRI=PCI(lvmj@=N9&RZ(VfCGgh;5ibZ8hrvN`;QvN(b&)Gg z=(rBm`cah9^LOlM-g8i@&y2Phm0Ge`F>*Ty=*{SJU$lPPT{xAz(RMkQ`U5MEuOH}G zO`k#zNn=XNHTc~>2SQu(R_jzU^tTwJeuP&nvwa1+rf=khz5&v(r|)(Jz7cM(jd%LQ z9P1i=Yn#JTL{T{Fo1lK1r1+q*ylsJZxE|JXIgf*@@!w`a*Rst!q{Xj~+%q8zyxdR~ zoKT2J(yOZsNs{BCb4#pjTcXSzw{+XjM585RYbweHlTiLJoU1}q$)~(CN`a@P%nuA@m zIPceW+yp_|R&;tRjTCVoTR9MP7Cs^p3>$sP9v9Y0YjK@Kgm)4QG*_V$w z6lg{1WiB~PrL8(SA^cVtG3zsYE7)#=nBA8cllmujCG3Qnp_rs;AtvdNoD_9(a~L3U zX7jXkT$~8Q`JPVB#l{^0;yj+CuMRo9(=WQQM{f3b0YPEdyDire&$|bEp8cz8@u6?c9f)Z_eJuJ{1HO@m|h3#RO4G>*^5Na zz8isFdbz*>Uu#iCDriBGlm)WXHt%2%8BcgOL*`NW>7y{zA90eF4;|ohdFd>MGJMP&{fgD6<#VgNA+ z<)#|I3z1so`<$U5)YADZ{{=ACxGT)5=$z*G0JdFDKLI_ad^KRkcBv);%(e`W*iENi z1?#y;Rfe7cEk6|FX=DvRK4?gFRRcnt{vSl3J1R|~nTAR_Gigk;iIRUTKb~$kxA)6* z718tT`lOFhUuRC%B9%iWLYkWT2X!l2qO?PJ&|U$ z>O5PkzHEvD&k|{z#4z1*&IV~`+V<6aW+^l3m~i@aAOB8c{(|x8RNp()ywn8{%=`W2 zN6t=v=I^JInz4j6vvvDU$^EWr_ur%5dUl>;)(F6D3)p8)bsj?{ew*}p!PJQCMUb2U zJ_`K@{R;(cItjYpzQU1~ux!LRlD~PvNspM~+Hw^b5X5mu$ed2)?n0~aXtyB27)x|g z*dns$Q=%hG6|)96I_q?r!l#g61AH`6OspJ_L0uc((KP>-Qw{GB+H}J5y-u`|RR`GG zQ*3`_Q$f=YZ&Isc^wMJM3Hl_F9ZiLD9z%_Qww2Z=(2gNmqR~}Q^a9hb=!tP-94dg= zC{$&fjvV2@PWP880Z-14i`k}A@FB$^ug{I|f4ej+`;%UlXEumHKdF@08w2RdXkrg! z9?rU;L0W9e@!~_8=qiKnhk1tX<5?g|?iP&QWx!$#@`NW#3V;fpYLGz*pWml3N z>xiEG3AJ#K6X^g5w#_YCVd@Z_-CId5j~iqK{(1*wd9V6m3lLkCi3skri^>bovZpT! zGGX_di?dtZ`x+(OaRpbD?ze_BTX>ZwrmYig$t4%DU{2nw@Pc)57uvv1Q_asEH_wLG7NYu8|mpT?>N}tE?_B* zz#Ba97e~l2s3N{GRQnACr6v1sP@-MWT3p~?%a1EciqouLX)lE-fW4%eUKXV>sL01) zmV!+M>#}gbW;Y?ow$i$ifc|RMUeN1JjGiPYBOnoK)Zo48KgABl znDQbhZD#AJo?AG-g+dcG+sP!(A7%0)JnZvT;FvW(CIEuhczfd#*06!Bi0jkp+4S&JiXQdTJTtWE+@AU??XjdmG5idGsj*}qT~ffX{+WiV=$&( zDS+VC(~;p*7v@ooH=KQgrqP+tP)@cp&*GdSfLB$mym$#$NqIy3H6t&^pmOIJMM`u& zf-1SE(v~qNA!O5)8zd^*i|GZOXD|PC>K;K_=Nw6A2b0NU)Ntz=^@WJovU0!06zyFdvks=iRq__l5l_S8nD6l#{oLeN~~(QUJ`ang?EDWty%kv zEsY@RO5+|D%r|*?R$mfbMe%kx*Dzm9G`KQ|&LEUd{Y13lJ5YJL996NB>6OfQ6DU7k=wx78508&k-)@vBU1z9Q0}7f{Xu_O~m1 zF@hO4Vr(nA0yR>|!wQ5@|ExUWcRsQCXfiv73wiRY=WLgYs{H*5>jE8OD(fLI(s zl`v&h&=IkuRl6HW1?rg(P$JgUuAoxe?v=PwomQ_mRHB#aa648a*8g5EQERv-7l6DO z%2#7j~$fyG&&ZuG|rCFkN6~K~> zgITy+R-QIjW@sot6(?Bks0?i`#je!LQ2F$+#n!9B@68LD5v$40y1pY_1(j|%lHW+q zE6YghgvyH2=R(PQq6*upuFn1ks0!NyzKZWS$beg>HM=hfp!D@1zMpydu`Fvo1PrD|>I+c>70At5UZz zIg0VK=Aw4o40e!k&DS!%7_P9Vf!TVGq0}4*h7c@lNy9Ck&yJL(Hv6`35fn4lofekx zh*yI5u?(V;XA)VmbDT-$Nuc=BYjJSqpRaG7+(@zyXtb`-R*y7ge3^?hZk zv&0H5L!^Ebwq6miDh!70MTE>$J{|Vr>qB+Yz*Qdz-kHgjk9@;?MW%h%m=rC9KHz=t zCj#sb)Z|8YhtKbW7-fTr=yQS26;o%GA4#P$hz0ElWa{Fn2@+~F#+r&i9ZUOWvILb# z(UBQzczG%dBFTbJ`h@zF)>D8HVUbu;EUJ($Q)P<-JUu~K#bJT^7 zZqxcY4mH{@EH10er!lwN3C2^}M4S2$r;1#7Z3F^0fT28+!wvxF>$+|_{jWbf)*6+* z*IQo?7uT&jjK-w(VLC6jx~k~hQ@JIM3`ztGolbI3R@>T>z%I4*i+F6BBQ9n6kK?J{jh?Q@ zS{E?GHGZR^$P(v`VtHp^FODrEz@1j;sLp_H5akxzYViwyNL1URv{!`y+8wDkv|R#F)c&}S|nT19N@wfqQf82+?%6Ui(~!K+avMs7iP zo?cRWM3w5;NbozGp)0IRSk&Xvh8y5qlkcxPlzCTB%np}tW5ccnRvAD3StH4T5eW*gIHDK&$3Tg<6m-8!$HSJY*a;AQal zJb}PTnWRq3Z+mYC3l#V7=F($6Wi2QZ3k&ev2Sy%gTUQ1B)!uB&w}$idl1s)zddK2t z5FNxc%D0jEQu|&7iZWMC)!Ziq>0V54{v>xeb6E9_rG*yA38mtL3P0 z$|#`m%q>bvt#@;s9)8cCNSxt@s=6%e)-mK+Imqc(_%Upt=Wi;24q@@;%p5mAadZe)L$^RPMV^!PCMNL z|HAcRF*%W}Y33>XPp?|WCHNFGokX>4OB2?mC|0;Vdd0ub*Go=jo}sE@PXR=oZR|s%Vu+|vVyN&| zS|eA)5>DpffRI) za$v_jp~f$*%5Byrt1({*Ta_)VN>$-@%3n86*S(Jui;JFiR5N&hi~>T#*y2;w)ye9z z0F9Z{pf^ZI=Ucsms!22{hbhci8=iep518AC<$f`~o?}*5I*!Olc9C>v zv_}PFms$wrh7?VWRpek>xuKpSJT7IWl`%9xfv7qTXk`}lf6-$Lqk)>O^R9y7SPEro z{wSsU4$}xf#h9X}ddeAFow2l6g;}tti!sSogqJOC|4D4Z`R;cO$Xz&~wM)V1F^W;} z^J?;L>S!0_A?o{U@D)djQU1`M4`MdL3LJKkuq)`sLN5b5mBsr)KM>F5wG3afecQ z3YDfWQ@b?M0ohz2x_7av#qD5R60w!I&B4wnj~a57zCwJC$~HsOEisD2#$C)tagF#P zlvy1};5w_msg6!AuRwHldDf-az&H<;IepMEhB_&d2-*}`YfD+Zf}m_lx{D@SC9rJs ziSa8*LSe6IXZnt=pxewpy`Pl!8`c(h?O62l47+yFu3?O1@xvGPk6D0LKpV1gPJt2<%GO+=?#_w+pMwB5xKDP2DEzl}Y`&gNOFvtWn%&A&vjFirq zU#^}-%Q&KD1*{tHqI>}$^7m6W`3sE*6Pt&X?D<8raX7j79}qR=>$eGyOy)mO_^}l4 zJ%Q@@p&S*lIaNYZKcEhIqMA9Uut%2wp$$v5nV=`l@%ghvHTfKcr|R2%@8wf#G_0OF zEq}%L$@^v_{rAICNNxOMP68;@)!bxOOy#F^kyC+GZowet`z5b&)w8?mpoTzor&=U$ zg@Yss=<}uM>}qaGUvdLehdxuS*XNn$`BkcSU-}Gy(i&(FJHN7aqUumnI0lc9GO}pc z>1elf!*ln{Uou=|9qu}95T4P$l1mFcNBnPoM)Uk~@O*lDZ#T?7M`5%raeRF5s(!6- zaL+IPzyH+O)lbX#AAa91kErCt9IapFt}4Lf~6Urc`V9iyIJQ=r>azLd2d-M{5e}t(`2QwQ2eC zm5L5eI<=Q5rE5}T%s`}1d+)q+0%O|^81rwyUNShmnKv&+yJc$a{SN-7+^}Ll$Q2N` zqE%OqI<23T+mJT6s;Sdfu-PPd= zU?c?`XLbFd%Gi;kPJgmD;O01jUNWdcxr%0SdP8=q_+V+mZ5Kd*3Oh793z;ccIK>Ij zNv4qpo6u$r3mDQ14{(>Y#IoPxsnh05V|K}^or;WvwWm}Po1{=7sm3bWpHf;6aD?>~ zxUkrsm4_`fWL#p$2~>cGS_gClt-XN4;&hVDZcmVJ$vx3_M&vBLq>pH1Z&ekY#r@{| z688}*9NI-*6)f+Jr2_F4p?umwUR4%$%aT9&yesSjhF0}Au=f(Ay@xIL^S_F_z7uur zDX$8UG)mV3ci{;#3`1$+G8laNX~&9jsf>Y;D=d~w8``?3B+*Pw!r~&lBg@&F0bowy z(_G!c6F8%%450~#onO@r;bO^PmM{3RJAXvp9Zt7kTlDVEr8_S!{$5p&ov4LGqnWZ7 z`-94}AXT7h-j|4`1#dSFos-k?qdZy@^Xif-p|UPvn{P_20xmC~9)rEeCk5{^I1Mz4 z)GDL%<0nZA$B>GpwZsJ8pLa{Q;gXUVK&R0XA1|UmR1CIU zp(9KWp@qWLC(}UPl6swUOST04X8;ExMhd? zy}w%cI&1kyP?@`l?^E~<`LjS3@`Aj+$s5g85?)n$oi{4567S{?Rmd5VL+Gn6 zx04Ju`ubvpNA%!FtePELq647%I%z>Dt5+njz-C2~xa1Tn${_WH(NpHmtof{ic zd%OE@h0)I7M81TV8Qo%wM*E3nYPDgFw+38oS=xdUP6~7{cbXwLp%zk6Ielcbe@J38 zm3y$G|LtlIq#+p>f%}5gFd7WA6R2r{*R-xoYbxi&62J}3J|g=#Ryj-p=gLi+N&9cp zLLa%?4#)kRXDR-$MNun}a#S+A_5H~Cp=yR|X$i&WK6=?X82iw!%_IvS zYgbE(n7}Q)`Z2pAEZ<`xwXyZL=uhDj1=pOJnu}{I_vR18iPMann~V}jG@D%}EMZ5K z0@xzn0ch|-J%&+(JKWQavE=G4q^gd#oA!@wgD5t>ZTSY6x>{`aQFLc$UB?$uPkh_s zOb26L+#^A8QaT6Wnvj>O+4?yWCJK!(9u9yo4tENmf%Ay#Osj2x?&?P@MMBUxivfQ_ z#&Kp!9RU)Df^aJ~(7YOkS~0~8eW;Yc5|Bi_CFy~bC;gc{i5b0!k^Lr9uNnY&Fz?Rg z=eRtA?R&6g^^ZCqg}GsIQ(uIS4QgIPZq9xQb<2>(JP}BKt zb@BM7B6o3oI)9GPqdPuf9(j z5=Z5}B3;NqF2m{u)j$4~WbPlsW(Ja1mz0?&T5ii_EOH#kDAmDJ>q1l~6NYIgiMo+X znW9QVgTn*TpS5RLx?8sF2EK-$nQzx(9|ZE=Q9ak@5R_MOws!7aLLA7ZX-u7%zf9ux zJ90E)VwPoH+`L1Q=lY$VUaE(50k+WyG-bWK-XSmos#zpScAJ_BT ztvze=+$VYS+$-MTzorMw|D3UTjE&i8m@$8hW##`h9{w6-f3cT7Vz+Dm2Y&iBPS4&h za<9Bax-$r7eU6%GwgOSk@hjTYmck6eJbz)uBuAR|k zj<>t0f#^l5Wr^bpZ5-#WwhmxG4>=%UgBaq_3k+JFY>gMR~INg*dns$L(ii?Qf=5?NAZ{`ux9=n z%EdDYs6C@?UY;%rZ&*o#B z6;f8Hvo!%iAaGAQ+2<~({2>vqik*$xS!q^CT8))OGL?w&y>F~r$!Xz>6lC{Pf z*5OjK*iD2tR{ysuGl4|fV*L%q;GOnY@%fkQMmHC=$hrL2thzQJTvYhe|3PF1Ubzxh zS*_7Q#Sv>=vt46m;Whtnp4$^=RVx=P6;OcZaEg_~cCFwvG}QC@wrh@qJk^Hy$#t^3 z_xu>1-CCJr|8vizugXqA`NzFZZJblvvsnl5R`sB6wZqXi(cIux6>msfnL`!dRteWs z4p%YNtMFENoKt}x+s)#i=TtzuMcodWylPrOdUi{F^mBlhGjW8)j}8&{OSo%aIp5+T zJ+=|U=V8It;R2HV2%X%nMsmK1)K>~_BYyN#156L|+UWwj=~=n>qd;uAB;?e&C(|~2 zdx+v)+qYK*-`Y^Cx^n6V+Y-k|eXq%KAge0{tY|x&y}@U2at+g#u+g>IlE>>Z$(A%RAk1?B`@uoJr-71 ze*m4()PkcjI$AR1p?&vkEvWvfy<>FOOs|dIY)RL}%mbsY=KX96G1{R0f&DlAZ9OR^ zf9U^JGbx2HZvx>|_*ap|=v2PH_Vbu3s7R_-RR^_SBU&tv2pBf88O<}XS`oRn7`!!T zmntKRQ!nMN+5ohKB#H9;_0L2|Jxi6%5m;iBS)p!8Q3?(7beh{{YO64i7G}9p##C%~ zJk`O2iL4q6<$H#Llku`QPO0x;+Eo5Fil-RSFcO=nBa>TR!&odfX>=sdV2cJ^b85#w zYs4EQ?i5W@^XJsXmG!I6KxL8-AtXpfjgCDt zShQYMI|-WGAwhJdBHipnPUhz5{C|%^$Yf_c4Ia>l@aiq#LTJ&MI*g6QrH5$CV}ua9 z&qXI2>yKz8=$S2zkADk%c|*{1nKH8Cia)n%Jer|~#d4WsO?C>b)j)*Uj}$U)-3xGC zoyys|Y0bGr zPs29pM{GKldx|`{Oc~Ccd#iFy)>!a?DT>YJERBW1h;N}zx3EII<2Tf*OF9O}KNh7S zgr-r@fC26O-R|ZItYGHx>u1!R5$$XxDzf(^h1l(G1|sExrf_c9C)L`!R93PWiPm(v@8!BcZ9W-Lhkr=ox{+7_0j&#(k9*|J08*XFwS?t7iP{p%Bp@4ZWaUVN6qSCMz_wYD#WQ4#X|Ae&n@N?t@9D3gieBn&<ot{(1!M0a#_9g|(T3l@yD#R@w>ZP^npt!7 z9M=C%|Ih2zCB!(r*L#HFr?hR9;fHam*XMVBVh;1L>ovmQ|MH9Pu@(DWqKtzf^o^aT zS6b5&=ArIC0LCeQn1`2HjFa2k|4!+1cmKF8jTfLSexke4A>-9}gTCmrhuUxWO8ruV z5N^3m!@)(V^+6_Kic+JVo!isd`rY=JJ|YMFKyTe}J}^$?{__pdj_70k;1|U*iL`^N zBr#^H7=v{a^nl)*d^!XfmtRT2DBR$>E~U?|bVnY!Yt544CCkGlbE-k6X{#OzQwXCl zc)&HU>dS@DF98MmuM5!~;j>ycT2~&JFC@0vm9T4J@ISY(gZK{@m?A+UdCVh#0*&EL za#0X~QE0XY{{7Kw)FncUDXBez#w8qO?=d05BuJqon`OEb*N~f>YEW6vqxn+`Nt57A z5mmFx1`?f6UG2=1&>3VlVsi{II#3rEQ1Pawv@TSP>Up>7f}{rBJCwu3+g`Me=hj?- z-XbXNXJ(q=V|=HsJms5j4S{it0C5a{Cd(ORBGixgUw#hdd*$H&?(F`YzdhRX|5*O; zo895-|NrMRzKt&%@eISUg|K$|G{No}p1(^U@tV0ZCvdzm#O1^&*3d_;ul^3-9dhuV zdtd5AFN6>ELX)`d=WAPBMI+llNnBw&$`)wxf5KLy7Z84IQ0+WEJ+&v@`G^qX=UwZ zV-vYMS7q>y4?nnAWsO;C^OPVWNcdB1j086mebl5i4RUZO(6)=Mh<2pQn)21-W zB5iOGXffY0zk_9OaAz+^03pOxbmgSR_{jPq8Vnz>QzF z+E2b%B*Tp+c?t^94%IZ=Mu=}iQX(hQvRqeV1Zh&F(+%%I_;z_*9TrsVWB~_8C_n!~ z7yZB_wA+}WJq!=T!mS4iRVLfbZmr-=C#pigK`we0WR5daUy7^nLSMS#keZP`99Qy` z*xe!6_LUg2*Y+gY90s(I<`Wr7Op*>>I*4nJ>^Axr!sE^MiKG0fiyRj8NHcgTJsmJV zf&uO&U#6$l9+U;;!@DK4aa6H|8Hf^~7^5BmEZ6wJfWMnH=vG10N@Ss?1;yiQhlw^* ztZmoCmo{AcWd~hLZ_`%|s-AV_MUKHRf3Q(q88Awhf58HbV zq+Y3rvj9pOs-v|Qh2HOvCCfA)+o-uTc4K@E8Hi~q(dDnmgf)!7R?lPFmb6DxdGqX? zM?c!rnI16dCb@F5+qWh6k*4B3cm4 zR_)fdb#nj=m1UwFwzYNocmf0;yqp{kierwm{+ zCV7FCQcDOSG;~{sz${>Tt3|UlS!9}uQ>9$!jDyN+$vOx#xEgEVFFGE5>nvbLgYL8* zgFOZdi&Wy4>YA5};2LBPjxcQ?oxM3`YIhaJE274GoNp4g*zMa}Gr(ddRLrPsbZwxEkSiAv& zxXiIt-*nC>%z7LOt|;=D1k|AFZQYL_cttIW+}^;pacd>7&ftfYb)FquJErmrTod_Q zS6rmXsszN!06$KkWZ4yI{V_y0%jyHiBRx zLst+n@cAs!0CTSy!}=jkQ#?A|5by}c!&1qE90racjfOQOVlIf|$FI(6s=_uxB8Lf* zXsD2DQ&GuOnA_uio6GN8@>F|_E85UYK?KFt;U^M%U4_9L8I8StOg8;+nwTBTIl0SK z#3j3$E4t4W=(H0F9ya(KWRiwVG&XG|Md_$e52Y%{Hz&#Gp#K?IrWbaOHyZj&1OqOt zl@Ed)fg0e2jsrt^8c6oOn9%Dt5xRW3GokY+uW`;%Va?ZK${=ZdSv*+rr?)7I?t?oI;m;;%aF#6x80D1zq=v%880&gN#vd1DPEqu_&C5c}Xo07R5)J(r1<6Oa$Xf)S zyy!?wbO!!+Qe6%8mpIg!7Dhv8V4XISZgR2IsTwxyZj8}4LCO2J5}!uugV)(r$ua;r z#SLmvR-b=Q60C*oV{1G*WOp!3se^q%w*r&xp|8+jzwMN)U-BVct^j6Z$ea5*Ylo+X zTbHXj{%-UcTCMDeRvQ`1Y~9SAaHjvP7X9V4qq(Y<$v7KMmv_@}252jSF$pDxmJm)D zbe#udWLg+6-p~bG$8U!7Ow^~dE9mAa=*J}if&r2mhCh~gV8g$)H?&=YB{FGyq{ASd zba+j}!SYn%a`ot0%_@#4b(jQY{KG_%$(*V-+#6b0^s=|g_mW>eSE1ZxpDwWx{MP6x z)>3_jFZ`*%g< zWj}~eT=1C2UjHqXd%@ZXGsr^-7;wn{JQ^&_hepe09Qso6#Ij;n8nZjmGPA5 zYPVX+qtT?jiNer5Z7SDEqXrO(m#)Wa*QsVG>W|r29@O+sa~XncVX?7@t9sME`j+bf z_QSNg17jDW$FN~6dq24^&dCnkW)rW>)2k_YQR5@Orfm?aWzYi-=x4G}MafeSDE++B z-O8bBg7|Yz{s^hXSr+XM%k6;M7`ctAU{9*!nMtTJsTC-S(wzJ~<04zqNa>r9Ie>Z^s=vj4^_I)6jD15LgiT^=epgYQhz z#V7P$2Mtj!V@Ii3btT*tIj(#z(RnCMro7GMelFPD`Ps#z+Sle}Lg!~17cG;CD_;Ol z^*;tU#6z#I4_wJT3A+v{HOZB?0|rdq{okBzQYw8hrR6aWOVdns5hCwW;SW1n*c zU{mwG4Wi?_L^Q^(FRA(z%2#3aw5z{c^Qn-35&Og&{;8dTg}6g%ADU~ZS;o8IYj{&f zQaI#6Dji>w5{$gw!I=k1>3KUJYLf1C9MsVal#n4=l&$P5=r(?dZ(_e=ytHjs{^12d zDM{jIE+Z8c-&ilNv_>`8sd{bhJ|VL$x`YdIK9Wta){)I37H<)(U`xvZ@GM6pDIif=wAXoWre1mO_)E9b6vp zcbH1XHN*zF(>C&FHH8VV8iJE}bqIEopjdb(-U>_=#}>-g6PGHWYTO7GrM44O6xh=i zm?rKw>&UVYibiuyJDYL;bQ%6~H2iLjF}>hpf69KRG2r+a_(e1H zTlIzYU$;#pr}kUB|A%~YULW`4EnEI9{ppWyV?U3vvwpn&pSf~(9|%13pB;Mt$I|)P zqiO3aB{}J5$a#9j*XF;ky8rQ}c>cHk_5;2lA3t7&(YB&~?HLq@Dxig6njpA*sI1 zwMf0A!=H@vI~LrQd|?ZWYv`IVRb|MdMk<@0AO=%u#56{yGgevg@Sn>jgJ5*-tc8gh z0e94Rj?y!RHEKR(pJr{MLq@=ixJ0D{fIP%>P;c^#GBD0L`Zf;V1(8lkv_^JJrNvsj zR%6!#SO#@&q9W3*-6YTC1T`5FSm$>_pVFP=8`ZhlamQcNIn|MYpy|6 zX2kqlVdPtaxLcf$sa;~$_fEx?TrZ2hz!lL&_4^mU3~}uGKS?_mO19P_DV$~)c=Z~s zNKYDrTbcx2rp4uTsJj6amS}{b%5YtJlH0mjIqh3NX*t`ld^|hcpI04J@ zYpLBrpYG@*4qvBt1}Y3BxG_mVbnL%zq{6lwt1sopzV9$;)wJWFY186DD11;7sp^_x zAmL_Rzmuekf#YVhpxNK{4ktTMK6iM34_!Pl{An^LN}2robfvsf+PAECCEFJFa>i>! zdMyvJCOS&6P4w@l#Nc$;vIwVP)vC#t>~C+j7Joe~XCYhF{^1$kZq(Df^~~OCioN+V zyRIS$rsCPjBroEWx6VH*l4WOWC>E#BR2b|l9mOMM$x_EVkAyS5@Yv)J<&P>Y@zGYz zMEcJi3#4HBE;`n?gB5l7czHMD$A3T3=%M2yJ{~!OYR{mM@pGDz!HQTU)?ir(%erl7 z_u7RP*kXtFwulZE7x`+gqi1O}q!}vnW~QF}gf2Pemb1?4xPD!$lDUt`uFzhu>o?v7 zH)oJ&3!#s5$IrrrZ^qUn4ap?Z^0;oiacJw40+SYMs0KYydxVZ#FM{a9dWsO(x|>#J z+wZV_{jIOQBSv_F{lW<>Wp@l~qZ#KlFIXXFB7-4rS-6BYk*l|LwURLo%`0>!C&f|% zSV|rG`k{olWa6#U$T-Dr%VsB!A7V9jEMUvu3415Ex0yYuzs=wS+z@ifPV>{*v&UDW zA&kJJI(XF|E++Xx;xd1aU8tOM5Xf$kv+w2^y;21Y{^NWa&wJDQ!+C>LV%4OFBqa%##(N;01wR1=#k?@;MA*7 zNKRZH;_p801il-ythtg$dCWFw(uBcKXO}~H+Y-GIbVGQ=D(xD9yJ;|Y22XqL)vVez z6l#F&IvBMlc7~)3+Nr`NuPpwK#P|c$a|~PhmYSvhu|jARZ7$W?*i=pHd3yEG(ogf$ z-#re2ubtGXzvrSeE5?`Aw@8n=otOOT!kC^#RCnc6M@# zYjbBflBF3$GHH7~m2_f278M`*8e){13X0MLcTDmA{Yqa=vDoMR<0yv5jMecGeLyB$ zPEfxPP;)^`cXP4LXHt|qGX$8lZXp(6J|O+!W&Z`S~}IPe*w>1{H!5{ zH?GgX**Q(2-d!IHUeR5T)@d9!If+t+9QLPW;4f>uN;NX3aE zE(L1sc7}D1<)M%aNR?YJlx+vUclDILpN5yiy)1KSnbfR%8ws<5SKEzVFK3Tb`RpqetRM&@SmyWVsf;7ojWqIEPvGTwL! z79QTZgDFu1HPE5wkpalCn6C#XSpZUGOfiz^fCI9P;I_}{@#1YFvFOc_REfIf81V7# zOWsUx`uDSW_5N+T&%q~(28E+oqPE=V5(Xo%8NWG--xdA>(?W+Jxyt{%JrB(WHVmRn zD5@T8q3x!$rmB!|rRSP30kS*PqFADi@fnoT>qh@$b7|COdJVsF5&a%S?0KllOzlN+ z4?3!{oWjo@foA9vMP|NYYZJ5#hGovLPwlXEn+Yr)|KRYVoNqeI&>6vHc;OlcZVMHn z@qXV|Oz8<+`;1Ej^tg~7=~w=i61Fd;o9c$dMK@4O55c|Fz`ZbUHR5SR#~>==}KW8Rj>DWD(mUld5l(&@*9Ot zhXiukOOl4#p3lyI{E0X)m3C~om#7~KPB^LcJalGHbTso|93cAK#aeH%x*U(*1l7(h z^|^H>n- zLG->SC?Cw;$M1%p;Mj<0*6& zuY(tdYVeA>SQS{!BlWeIZssNWk=(6DN7j$9T#D9fwkBaK@xpJq^CD#jgu|DwJ%Dhv0Nq{ooaSn1FsUH)Fp=OEbT@-(=$E_}Uw z9Ac8CkfIe~Sw^F9b?&n!`syWI`p}eS3kre3s+UUk%e3&6{7ausc3eq_U)hvB7Ve2m zA1t)vI#)cNIgV+F?9^ajuM3lg9vC;>`K}6G%75dD=ngGh2Xu@&&(vzX#_CyT#l5S# z1@`X;h}&%{8|JFYhALk>Ws|u&A%6G$b^V>PnLmBr|NY+m-7VPk*hDG^?2p@`>0w*A zBU%;?miB%P?z>D%vx8XxPfs4o4Q_Fi2;}&uQz1)^>ZY#HhYa=0(mjhHEiZl-DP7wZ zH;e}XX4#TH)6$_6dJ6iZdzm%RBdo!;%cnfBU~?>>+HI^M*StxHW>1Qcr<7sv0}Yi-Y6%hV8F8W%4yse{2|+ueh$h%ZT)Q&Wwv#AWfvJ}b zkqErY?W>;=S$d_8lv%wVhd5VfizMU{G$S8gqzb!CRkMP|L;+T~R1<>V7wc$TG2|77 z%r`i^@5cYLG`TGmK(Yt7dqly02ijx*)o&kaHjm%r*8C$WYGGIO=m__z4zCp6k{eQC zmdtqZ4}!lc;SF6kR?g2tjb;*g*?rvj3S!&vQ$fA2!eIf6z@-1iL7<9$XQW!f$Tnk0 z1^ZNpmSRNee4YA~it| zv{Awc7sac-`dKMcDm3G8qgBU_QCjLEIJ&PuR#Hf!Y0HsBJZ^_VA%}7&P>l2N4Vb%V zy_M?Z)v+KB715IubAj>Xj~bMY}&1M|ZU=hA&(*Bad^ zO4NL8D=rub@L=&qW4fPSG)|LCQ-7j~Fiz=M8J8;Qq{=~5KBW#jd8@|GBrO`7G!z4g zrUWw6eW3vz*A~T;4OXtjRCJz&&VvQ8ZVk8ru@OrD>kkmH}F*QSStP6 zp_LaKRp6*xwr71>05m5}SP->ZyC~^^C&)wcg zp3K!{z)ci^WAg|$%x74EvNjp!YnoepCbsMSZS#e=gKP0CV9t5P72` z60$YPH8(m7j#9gQB_`-Um|f%m;@VVEL)JvCq9tYr5^Y5uY|>BXN-WF}gOX{Cmon3X zurO{3wl4B_T|Q6`M|m$e_zCUOw>O8u7DOV3M2>#k;^$dq6zezq5* zo8yV$CRvV$ZCEeQeusFST;_d4OBEAc{`q3G)Ps0v-2|IruOoxu>K+EK{R5P^g^mMV z8-FNSxLwIDjH41-S$d1)B97rKC)LaN2d2*cJ?VcxAq1y4uhi9ZH$_QuIt+M<$at1q zmSJkyLr3@nfsPn8w-oEtyHh?yn9B2`4luW#`~mXvB6q^YXMB*B{=r6Hd}nMqTFa~c ziI?vF2mW62h*`g5GUwACd0V zP8G~}O`+Hv8MhXV?SNe8-+cwbkDV@#{l!ZyW4cUNT~N0Q57ZS z7AaT~vCCaYlx~j1TqTNuyCKhxrTsS!r98rKfz85+0;X$=Pr0eFuZWG>*Fb_|Z3K#& zU+9y)1Vf?&M&^$^6d4EY|CV7L_LgrhVkVoFA~l~fDCyAH&-a5YHXC|>^qD=gg`YqK z8;_)r2735+#&cc|Gk7YO!RGw2S&sqnO$)m|U&%e)ip(YVEm2=d$%Qd%qa~S9YZA^4 z3S3x?|JqC!WJNEH&R=yb$aC{|1lpin0`WVkj2_CObdzSwna*g}r%Rfnet;{@u&EXc zTB4m`SqipTkI7`6gQKCvbo%8Kb}#bxKdNB_0Q7LZ5d&{{g^3_1u$LWA?`3Q;SeeqT*Tgsb5(Lu{=cl3>%V-SE}QZ_T~7L-t^PvXQ0h^a z$WT{-V=O_=x#eStZnqu@<1h|6>6y{dSWB9I z*%oLaB=0$OdXMi{7ULt=vFWo~%LZMW0og1pP4TiL8vyGs3t3c9@@I z&uT`!9sUJGDC{d0d1-L&fO(Zfj4PGoxKzI0J(d4}?X#37iQARm5zK$x+<6#))mgqt zRjnv^ePVu+D8kkan4y$c{i4L?&3L%7sL1i}wcOd#StcY4DZ?SqY{mRZgr_Hcx-N)Q zhFB#zl^belbvlgcDG)<8nHS6t0X@GzDiHU;Y5Qxk%LX12dwGl%d2{N3={^G_0qM5z zrvL>e%?5G-%ZRBfbeYP| zw*8j&p(L$E&&=979C|x_X@S3tgFmA2hIz^sLPSM0n&g%a#RX=kh&q0IpQG<;2NW?n z4^cBw-+oOMP_4T}S#cS|e_dnR;u*`%l|+dtb09u*JB-i%!_izG)VB~| zbUsomY0W{X%ZZO7duO7mzl3K&Q*uQd(RqaoWswbXT} zA3LrWFY3K32GBq1GdvM4bDF;0{0O{MHU%lZJannfJ-hsN(5KURJZ{gUWxPj4_Cx@Y zk?=%@GaA&v%Y0+kJhw=gtFlS(;gKKV@9p*iCc6QUSF%(BKP*?xcNwsNgCO^;BcDjf zw5O?>0fV)`qtA^PNc`0fNO3Wuh9(`PiVQNBrxs`%WDPGdQKDl(0pOT5%x^)TuWncN z+)gz2lffb1mYlFxoXc)Y4S2|FS(bn_dE;XS?AUh;#9olQsY5w6u2oD4=L^z2xO; zee&Vri&+ld8FPkq7t48yL)m7A)A7nGT+Wnf#z)TD)Ez#j#8@FdKy$C9?!_YXSG*3) ziOAN42qtg7fXkG(TMEvDBboyt#|{x|a0cB3y5h{!7n{LQkl-T*cb60bcQ0WKgcTr| zn=cUn$6^}+iNZPP1&6;=h{ioI#O53!j>0<_g~dPkhs`r`Z2LkWRIT?H1kybYsQ^RV z`-PuR2og&VmXN2AbFQC4@C`?FBruwo0Se{eS6#$+MV!hF!okIXD|Lm@0%p+w^<6d} zPxe=|tS<-PtNQffnD6tJc^;w&&Ajwc#5W>#k#-BZuen1J^B&)_@uU5>S--^gd<4e;LZ@;3Mk_sMzUH>X z;x_WCKx);eU!~+hQNjsd9fK8~&x362*6A!4k4LfC^+R`@E9bJbGg7?(_ZY&TFtOKI zQGC|Bl6Wx=`6zCS0|nOUZtU1IRbE1xW67b z4ksxk>U||PEE3Lp-auPTFO6LUAvJ7}wZm){ynS%R_B$7)SA3B0mwLVwi8bH%&1!1L z^qvAIVselL@2E_yx=qvDE-AIy8(>1ri}^~)vECRR0Y(DA5tgd?n@%so^w-zncPM;J z!#la_e;Kiv7h5g~p*%X>1Z@8p2=exfR@PDo62j(&qe>;xV+Nz&2n0ldxG*2Tas9}t zyiq|9-aP%D@x;*z4u1VBT5lvkbXJmPeCChfYNU9EOR=Tk;>Alo@krTqhx*%^{)L~Vuvc~eHQtov}&397K1T}VSQNSkCOuULZ z`w|cFBzl{ezC%i|W}O2iq;%aSx7bCC+FcCYZ={dpylAu0)_UYME~3}#`?dQ0{XZYu ze|E$ezPOyw^p@G0z@fNC)QM9%8Yldsr2%2K{h5eexZ($7@5k& zhd@ExYIj`<*ViyRmh4lkWO`yuGN;P*QB)_8=XW}h@V-+=0#j3uLAcqO+HrNHCNxU# zO6_D|#7UQ!gim}=JQ2+8$aKlf4leLwi16mewzx~x)PY0=GvE)q$9Ib0s(u@qJ83<9PD;A+Vl5{4R%hN;6(wl1*@uyOwr8dwWSd0U4PW zB&bXD#&LbR?Z4>h)HkOIm5bYA4hK0UySzHwM*HMBhS-+44Xq4$)Q47vd^=Z$5`go9 z>o9~n(HvoIY4CZtjrG%;;! zOg11bqAt&}m&xMOq6mqt>0^G(G$s>MNL0{Icb66t6_-c0&7soJseQ#T&2Z{_5%@>X zxpO&PSH|wQ=@Yw21_TyPx%)9@kW7Q@uaN618D~-r5hfgYB9CiqzLu&2B@Sx(FXacW zADtruy)f7C=EDiE4d~IDs%sVB>60y1ABS6$^VIHcH$w!cUg?H27P@wEENZR>fqYXB zeS|P44CgCxxQit8KePmI^4pYdeKCcIG)&uq(gQYmf}FQyLm~HgTJKHRvTNn>7Gh}) zmxBsq)}EqiPKgrzril`KT4@bhfD@^Vspdc0_8Fs`HQAuQLa~DfhX`rs8?sx9(1x$K zJGpqesP=fW8So3x*7^zIHohAZPD@)3B?aL+hD8%Wd)s&~aEeyL80}xAk1YGfMn`E? zGNw?9M!S;s`7-`?d}n{B`bqnqehY|5Z*>m);<0Ss)`N`(o0uR2b1uUYaq(bW&{@8Y zpjfs?0;Qnax42rN!5al1$h-54K~m^qIMd-a)R*v;};-(08veanNT>^wHl zYD1({3+hfH<&XQuJCx7z{!e1@vngKyD%?yoA5V3X+hyOKz)<-&s7N~*gsLv+Onek& zo|^KY(G-Z<7J9Z-UV?#W zrNjjCT3PT1IwEo5GsZ*-{G%-j-PJ#tmPB=$bRYDs`H9mQ*i^8orFQ#N^(26el!MfCRfQ8gQe z(CmP(LJky0HWr{~v;zZU;XiekxEzuG0u3EtDH0CZ*4A&S(SMt`C;5{}*N#)ukmi?K zUhFH0u1RV5&w}Hi`}AHh7qy^moBtQ`TuaZO6Hq15kY_MT&@R8CBEr;oZjTjm*@Kas zM~KBz=?@AFpTi7}!Z@1ijoEcjq^&i5Fey9~uKZZCJO0U&MQaIdFd%Ys&}55(m^ z8V*q&hTbqaz;hX~Jk34l%YMt?OySpH0SdQV3e{27>S;qCU8~=L-=C2r^ZbJS&^NlmBlUqeSaFeWq_s{y! z{Qdp;@o|Lh%&+~x@`3F0BD$i_UxK3KyC)>LnL+vg?3|E&reyo1M+f8R?2li3^iK*Y zBmG%h{lQa;uYz$RXtmxJZD{X+brG=0;eosyv*hiW?He3dRVLeJZaV9!FKV$XDKIiO z>bBvH`(hUSX!qWKzv4C^Fs-F6T|;anI%q$5HRsH8c3=d^&$M#$bp7dsKFCy9!3zHW z&1r4^c1O)TM}x$JpILZ0dkUSYH?u+;=|iU)8roWVJN|lY&r_e5rEk6H5<<#OGG&*# zW3AIHbtME|QeY#e=CB|I-n2TTu{13LIgZl!`SasyogEXp;Hr<0;p(9wVPY+!}S|J)zU;Jm!u=Bhf@V#^T0 z5NH*Xf9Waemyscm#>-EaZ4QKGub_M#@=*P~HK=b$f$rx?+IYKp>^cI;<(lG6n615yq#7z{j+6eae&n_IGpU}wl2$c(^RMZ$2Fg^7cS4>o8v;8@v>yQ(&XA*p& zD!*|j7YfBHo!$>E&HnWW0r7(Hay+#94kf_Cgy)X$pOb@mKN2c&aa{)e+i5mkJ9u1xI8KMSaXE*D;=bnNHLrWjMvBq7)uvB#?snLI zrDj0duFG-iV=VGWm2}x*C)ri#@z``v+oD8y3Al^)zO+8JWYCP2>9gZ;f!==v;)z-Y z`3mpG(Aj_VqJMs_+V8yf>@sa0yJ$S~wsu!Cz?b!2ZF3@tF|BnTjTwXAj7C?vsz&aho!H;ztoAJPiPKxk*<{IDAre~x*$AfvDvf~K4rGEq z3u~cHlQi*jWx6<}Jq#CV&jea-G)<0X-Mw9{k!`j0QY<*X;5I#q$u+@N=m)(NO_gf# z;Sc5dL1jOqz_v(PwXWr$t!dT9ndsruwI5?2n0tY(1v`QP+6L=nwWJ_hazbPk)=A*` zZ4W73$Ds!a<*EBPPh6n!UDuo1h59R(&kv>lRw_*HpNng zO-(cJNhUCh9tNK8vk=Qh@uZO7yIaH3=Hitv-9-u%xz5NrKM&2`x_Lh?zV?4WI7}vhP!!#R+O6xjR0%XPOY6Zk`$@W@G&2WE3l5;C zEa%whayfYNP^KM|EJ%>$_RZZqeEvgr>`HANLm=~H@eWrO?q46CYdiiu1~xtmmI;~(aiqAG9sAYl_V;U%;G)<7 zP%w^ZW9?xu;z64EbH6i}{w#ixR zDz?~i^BX|!%^6gAz#{N!QLi1QRN?Y#|!dElR=o~poV}us;4w%r@y77jQv?q?D9K;LXkJDvyjREKXS3-i#95U6;QJ-`nU=LmwDIdE{&3sVr_aX7^TA*%06*^?|LKx+J~+H^=} zhJ(gF%8h8{Fw8ozEHB^&AkFy(c982N2;K}GdlXT}N!}ET8nl7x_FrGT%TH+4KaQYv zi{5R;VXv417m9-&f_y|8Ox6~PR6B8M$Rm=y=ti#-D0;Xni2MaCf45qo zpP}nv5}^ox1>WzY@&m*#%Tuv$nLG5@15ZS7{j2L+GFak}(Y9duGXcNuRiu#$~IXo?9n@;b} zfVNvI%0wuh*l~^vPnj#yEO7bOSjVjgM?Un$8c;rYDfF-o^mDZB4cXz1HXZY5#WhZh z;tkrNlc6IYtOTg+Kl8677#Utcl_B(o+5^2^o~~ueS#!f5^T?JgWkNgAN_N;YGTDG2 z#ehIwMzym2?YEqFBcMzt8U{7h zv22wUY-z(fKBlIs@Jzbwtd9D)+jyq*69v#lMZ@69@6p_OcX8~Q%vvsc2`|}jCaWj- zivi@yGV4sf|5K=SXp6mnyl&+g%P}ke&(f|x(T!bk??W2{m_}v0265M2)50ax3=gAo zg7N|z1P1FfTgT#LirY?K&q>m3m^%>^{tjG?oaF5^owe=JnmU1s_i-o?$crw{K{iPd z-0n**_S({cHm)JmuzDZBG>UY?V@O?)X$W~KhJ6Uxrt`-&OKO= zZo7dE~JUHafXw7Fq2+D!HZ7v|%9r$LHEI3t#uJ zZ5a6P)N$4;a{mX|cjeY-`cN6aSV!D(mRal_W(9gkDnAxx= zwNAkSkM$etbvB$oTF6%Td-vrDrYfuh<>19kkrGWx#ESy0Ro~Jqu@fAsO2pd~=?4~% z8Q)xmu1At&!1<2|ya7%|u4=~+2Dl?R0@6LSVJOBW+S>9bLKe{tU$0HLX30)P5H?jW zT^2H9$kZfvSzlPYJ)z#~Td{N9!=!riy!L(_FtE?Ne*wln9b&#@lt8M-H%22#YYb(PM>uM z$K?92mNvxWR>YiTN)OmM(Z+itzP|v$4U*;3dOtMGczo*lh+^Sm6s~*3g%YLx86aMV z2*?NuqqNyWG$w51pV9vNo^_n8gE5ENWU6s5CuM?jwnt%-w{R;P6RqW5f94VIbFWN# z1r+N6BqsyOMKY;SyHby;Y-G+ouQwGNU$371HjgWaGJK~J8Gs%c6&d~qzdV`mN-lE? z_)KSM2qXMI3EQ66Ke@{7sDV(qm~b$KVPNsR5N(ZdMF!#Jli^HxpDV>2i@vjBl9zGv z)QGk9ZQMv3d~VSZin%`HRqAG;-?y_=0ikPE*qW%tIjQ#o7Tm==(srTZ_h{2O1V9C< z<08YqcJ4t&c6{6hW|$dkpO?R;PkEkYjAcpYuM7OUxv_a$8gvAPNrzjiXn)u*lz%4( z?BoedDdI8W4geWY)PXyy>x>42A0&S|*n1!;i*y6F@V5oz%AjxX|0Jb@KY)IKK?eyj z2}$qavPoT5mFkbf@B54`B*af3cQn;s#7E65d0eEJ%?$)-4HjG-kxe3RH2kR!h2lUk z_nE6D-4V1-GZtDCb;^?MwnVRh(h!$q_8k0bK1B7a{0=orq(-D+!59xx@o`K}*GnA= zXOj2*PP^9vFwk(?Jjz)a=^bwz%%M0mCjdB@Bj&TSH)|g2B_df#JBp^K2I}i9d=Br> zCXkR>vU+W&O#-prg>bN7=WzsU9~kG%3Nvb%Q3ybsX=yShYZM47u^goa8h}gZi`okm z2qW1ojVJ4aZ68%BS1ttXcn-GssXXojOXV&Cp>V4CD{r!l>uz2;aIzo7Sg?@V?~{^A>&TbxW>oao=8H2S&~a(4@8mkkud3FUt(^?KnF&mJlxQ3`1|uVKNUmwO z3NHu2SYukBkpH2Q%CBSjk6E~h8k^fG6*V?jPTs*vDpb(0P)ME%(m!jODpYw1?e^+| z`?e9j0SR5SV2O0-xAkTX1h+1tVSPSy?7U2e$7bREUVK*JR!`wUN4N_3jtTtM1^dnf z|E}!sJ`b!$Xey6vXfz1y%%C&%37->su{ z8clMJAGm7Opynk|(FJX%PR@<3Ga4dJn#^x(XAO?}veW}YHI^xObR*~QJSgm^5A(QG zI+zj+)V$u{SFl!}Y({_Uk_WVQ_G!~;Q~EQfo3@m{6Is()>|NSbcx>zYJl%fPXIs`( z=YL3XP!_wCnkRYb`_Q>eXp?pBb*Z&*j+C5*YGvjko9`ovrH^#*u(p$A&@@OyVA;Q7 z9E2S7-4r*Rx1AC&b$=TX-eb?c)f30+xws>02uDta80dV=qG1(L!hFuF0)zVg8goiP^~3nNU=av*>y%ywcv+AAp2Mv{Jkh zyw^r5IBp|re~Llu1w-dAHSI$KwC@uoyTakttrz?o2_vH}m7z|33oNsl$f+c&@p;iA zGS^P_L}zkoU#3nYuidOmf?HG8C!g&#((a0`iGz@1I!XK(9lh4;zS|bTEC}%KzZ}CLP4JY*fD7-n zS@sazN(^q!SKz#t`-RQ~he)bq)fP>2`JfA98dA#UY(fkiAzJc4bwy0 zYrsS*_yZZywBX(2U#}!vQTJ`3lNu59%{SVL+ucNvim73wkl~-V|J*{UAbR;Wz1~hI zZm#A2_3-_Q8g>>P_aBv*RURIm#?H;$s!%aYH4UwV%a!%T_23_w7_)dFnK;G5>TB0- ze@7rhqq|>6Qgvb(VdiQP1NS>S+`bU8*7>|Pna{JlL$0(+CiIp(D8aS;HOAYCj>&o- z&6HfQ;GLeEHxZ_8H0Jue*PLWKS_{7ilUYrfd{&`SpMfi(?(hvxy1TqC#dUq(p0HKx z_t)ufwFVrZ_#`pgqUbvmAo*SdMqdGh-F#;sCjtBEw~CIK?S-GK>ct^R)U<_A;sw!_qMF z%zRtFn-)~;|Nc|`RN!mOH#=}uZn(U#Dg@_xm?m$t;9H+ykhcjEKHr?dr?^q}9hx)78~NmqHK9{V+_^AD5aQZT}t&iI!; zVWyRlKtzWTty;1K)j&F+*l84!7Izn|4t=w)J>$4G$c=!#;+p>NzvBbt*mGF*{s0HR zL{+fK^!r_yJ{Yq+(-;0{;UgmfltD5825@ED+{+dbQ)Wy&X7jKkw=d9}@?rK)XyOLf{^ zM}1|)d`8A{^#fE*Q^5hN1f?Giq9LqDBxbZKp0^4Pdop%=&^$c$rayK& zY6o~UFAFQU`sRwx-=sk7TyGe3wW^JfJScRbGVC_j?D*m0%*+YdR#-L^pSbYS5PgVh zKA)c2ScbGsOiLYBmzsY(8!O{eQg((HZ(S9!cvq#N*zbP+lC=~G=1mYnJZri+jpX@W zzUh^s?$vD+A}x^HL0>aP?mdn(%qHsK#QB331_89!sgDGkd4hI%AHV|16jay!3?U}{ z3EL)}!>(Jtu7=xz7EK1$3}6rBvD(Te$v5?Oa5q+D70Hw16wir~g|svYxYD->lHl=nH~8p44I!<%geO6u1q zlH&B54XPodL#$>`yHNX!XifvGdYjq6cm6cu(#ugRcqm0dJJ`&L1xwI*1yPp-Ex;Ze zKiDN43OvO!hZga}Es*F%O%V~bIQx~+jv+DvgZ{`9`&q8qOHiR}8b&iLNFuu#TV-gB z*_GK?>%#_0H4R=*QN=E8v?A2dcJOf=qz`drRUx5AaV~-gnbD4v1h)RT~vSg8}tui>AdG1d}e_j`Q5B2 zF+Ky!fyHmv$FnB`{<=8;?+Jm!>$2s%uAy-+@2uGhjk_E5-4ia!?AdKo#2S;zgGv>I zsvSVvlx8io;p?&eJkq1Q=L`s0rW)|Ym0SCWx?TS&Ikuh!E(c=Q2@5Cpm^cNA5l)?E zCfS8O)5I&xhN*ra3U6r*%rMDX%fY(iSfj;dE}8zL%Vegq9v!2Q#=yr1B;CDQ4>Z}D zf2m&_j+LY%W&EB~`CSGeuI*Y?Truss!SXMj{WHB0{*{m_=ee&@eoQ2BbJmh@8-8a^ zuZQZ#0ED4;0Xn&c`HtD?Bg;4-n(ct#fT&qSa+vM>?c=$H$zq7CD82r@fi=M2lYd(9 z60XbD4JE_s4uyhp6Z%}ylJZJBwaQ*@`A9|&F`EC|VCr>PbjFiD*spq`p%|;KGv;ZZ zix}=ZEf8pBGgNf#(~LHLVBK?Wgi$-s$TZzz`Zt`{pa|-yBI>0u3Npc#RbjN&(so_) zN<^F{Hoi3EiA>;4JO_MaTBwdj34$~1M#sEn!t!eJACJjyK7Fgs>U}ES#eZfeTCAL`*>~7~@4qWFMqZ;`6VDb~8BC3PQ0qdl@o3KjyNOrPBFQb!w zS5qG!%g5-}Y%q2kU8(#H zRf=@8w5__e#E1m4QGlNrH*tz!tn|FInk%hLKM-bUk6u1=j~bw$g#m0CGuJhdT%HAq zf3L!`!M1|gfp}DobbQ$;ZKM_d6qK8motmnEDB_19!Xe-AD$zapDA|1EU8h`q={*H= z6hWnjif%2ZZ+C1&#LI5>oL@oz z;&424lyiV4AVM&)W(VRElD=fj>X+IkW`6^f5VHA3eNTuRa0TfhGaw7QQI2O6)a?Iy za0FqxGR;aZ)z4I1=&k~HwI#0NXE*}p?TbSWyjm}oF<&}dDuov4r79DRQ!$3VV|qeh z7QV=`cat%?AA!nRn9g9WZ!x9`9+h+(aop>5@%i|qpNDq1f7plp+K;U7&iXX?zJ0H+ zpP%pcy}vq7FaHrEH|_q=Nj#4hw5(a9gS zq)xXRQ+GQraS+puKW%wkw@4QIxVwM!V!r`_4YV;ZDNzwmX(9z9S3kURjtz)Lu0X2> z6GSDD0WuEc)U8CP_5XCvAV11OV(V!gkg2SaarMKi3E?v~v;T69#4KS2%x#HfgMo_r zn}$Rg7``9q=C}Ik&w65j1Uvn+2SWRoPZ+}8)A*;x`+i8B(*{3X$@p4p(6~tI)1Va% z!ZjhiX9Hj+XPSxF6d;qvrEdc*_zQrk>&e5=j|slTGV7533afDna1)sWzK%@9Pm$yuiC@EFJ@_wSM z*UZluji(5s2jSxemr}XopVAn};v082^vzkiEuf2n@y`+RV9`@vXc z7ml}7=dw?Tr;WJa>wath7_SUQXB$ZqopQ=}6@@IQL2J-w+^%ILGhRar|KZg@4_M@0 zFC3Vl3Qc(?86PYAW3H{q6sY^X|Ak>0=s6kar{bgi?zF(d?W#9(h>Tun&|^&c7;<~d z@vgM&qzWdAA1bK<4cJT+Jg;nYyWq43w(N z4^o60?WA=Q1Fhn|D`{>jeG~U@UDz>!rX$U!-pP3Q)v+7dXuk44AXd5pO&4DvDZJr9 z${J?6qIUyR#b-+aH*=d|#n|8BfTl2@XRc8reu4MAxH5?eU0fBF2ov_=(R@%p0WzckydcE1mKa18%g6TEu{C>2et^A zzP=@EABDA!cX92*S(S`p#Q313uEy@y8mT8YH~OpGavftF?lt@tfThZK5AtsF8P5## zb8&Tvh$lunL|71ezq{Z&1ci1Ide8zBTx7q82Jms6s=whPc1}_UHCEO>~g?s1GW!;kb^QlK4aoHnoo_l9ah%R%+sE8+;$@vfDms?c9LW= zD-(;i>-MyE?@wr38)JMvaN}4GipC@2yYx-;}y#xr=k;3%71`SVQb*W z>rgAqD`_*EH2j5*_*;T|F4aE%$8g|VUjN<#X!Drl0h*!FHV zOoH4LuLT{ZI)b;L6}UncqFHvpL?t3#S1BTlR`cTHH-ua;3n2b~h=8`hJZ?|LcZ4xP zV5%0t0j!k>SI?YAxA!!Uhg4GnbdzSZFaiD(lyZELM4S}$B=FG!mp_+NZ)r9Ju`?%t zSvZ02XrbnVu0>iFGNlENz)r>&L<(!vPSNnL&zAKd;_VDlfemvFJr?ak+g8itt%zCe z`sD4@JlQ08tT*T}h1`&wF~U==K2TARm@mxLrChnyzP$<4#C}`>0%Fw;nii7;5Et%c z1pjA3gRKBM%Nf$hb(h4$^V z@0h;^jg1T3*$PM?;Iq@ev4WnF?-g!<&LN!C`mi*TNK#Y zJzdWPB1&kBE0U+;m(=2W0-_H)wRQ1bPtXlxy6N@%2L(s~5KS zOkFO>2L=i9xY=($XH;l)^eyFy-fFw$l{FkvuIt`j&spNmte>a?8L7FAU$~u@Y!F1K`Rt-Ej;5yV*wE2Hh&DSu5-3sgIW9<1^Es5pv0G4_ zF;%`wDEb5rVdQT1(OT=kd#Ccc*y?D^@9fOW0gO#4 zGFamu+CK179=QtNhC0Rz_?)gUNu2XLme+CtWNwlsHzFYQzo{FHEy@_scjZ+_EZMlq z(r8T1$Q>)F&tv103X+w1CIRwmuk`+2mc( zy84*)K=3Nwze{4_s%o)e+NFsUOT}XkH`Ae2G^nX#^uuOWn(-eH1!x~gHy2lzrnMRb zqrn+sF;UTqr4&Vd?uEKwh?87^O?og-cXv*l7-^1_CgMKVoG!He}d2pjr#k-0jL*&R#H6<?DUX^ojjam=?fUW+OJH!URWuS89E1I(87454_ zdT?5R7~iSh%$=xV7^I{|5pN8P)H_>P(pgGesmYur><$rgdRB)D*VYF%l-3>QH1a@D z`tSQ~EOWsZ@HnCD!UJwWKR{_o3=Nblm%ht+s**XEnw*|?3N%d_9??tp= zOPTYZhY0IbMKUv64MNOWRZr<1U7NC>R=!af^bLrhaQ#=h_mDP^Jpfwu-bp1HwqOTR z$#BNNxNStEopG3zg8x*qs`$JMH=mJ9Pj9Q^I628-LTFN~WVsy53We;~Dx@i;ChLk& zd&z>K2<}(Rsli0CN}Qva=G!^f3Ufe2c1?bsx%7!F=PAv3vIZdz>K-j*Ze}J0;W0}y zS%6LkH0uWm4Rq<1R~mS8l!QRGAU+ON&ljBK>UXfC0dzRPE&_O(;}K^4ux#UznZ{$XPDhM2EO&Th-ta-j8Ws)~kb)VIeigUu8y!ygfoX*3?w17I;dQXHly&F4w;dD47Fj5QDgY5W;D9QM%@ z?vejIAm-7pS;-E>HX29w2gfve6nI7tg=Mrgj?u9(jKYPGQ0$^@lj?#2SjK~|j0RqL z)SxRn1Fnp}Adi7f^l(TcqhRJj@ra&|!Ef&v90_me!Lf#pj5G8rIQgL^?t*ZIw!jn` z&Se>oClr45kH--jiy`EeA%I+l@q+`*it6(~h(5O2lKoyUNr^I#}C2Sm_$9OyX@ikuU+P{&5h zc?h(eQCxl)C1)G9elT>LLn7mhTDOtcVHg@tfatd~63*Vz9^udiCO)3LW?9Z6eYw1B z#4!t|?_JHePdeLroXLYUDoUDk<_=aHgMd0KdMNJn-ztYt3kE<)7V-jbkD)!^MqeGP ze*TCNMBW{qvxxCKrdlp~U`j^51fddc+WMr7M=2OP4&6#cb+bF*9c2>3cwr+%Fw_=5 zF~iWJP{IlmoEHu#K*)-UIMFiL5QT4t(eSKu(-Cz;8E!Ptd0VKM0#qU!O}XTEOz6`( z`zXF)CTez|)rExcEmcqv(>x~MIJ&m}g|#T?=b5qOqIN#Sb8=?WRX>W4#TVpzG0k$e z;GYrDM8VG=i>cKE$PjhH4QoSXjQWbCIKv)Zm8y__P*GZ{A>&l3SeNV3jTV;BiE|&H zvCmY0F^gn+p01}6bnCo>AD?{`7a7fmddB3Oxx91cjwwQOxOk#cxFN_K;-LnbtZ`a1 zerU-$?pXX$IOG+5R#y7j9yF@0ZR({~RrO+vkD~3Y8}*D2WfnpB?812V0p>~O)w$A4E4Q?84l9uH~U#p7%&%Htyo-J*C0t7 zwB|*b1Iu<_%LNXtlC}k7@t}#qBy9^BvyC~o6Ox|TklL%Ktq+&JTDr++qr)93LAPF~ z1v>_V{bcueNUsD6AU8P`3LsnH2dzDuBqAJObKJ9!;_aX>!0^JCTGwZvGm07k1#wYd zNvXb}9oL?MHrhOrPbtt%dS5k3m=`GBcKWm<@tCoEuv9s6vT7bgu3qFUjgSZ_2x z8y(|Li^L#R*>F^)WW)ib6*i>W4qUuS6pF#kR`E$z-nS5z^y>kW~|!FKt$jXJ2)jYZCN~PQxtU9@1&7 zsPR#7Re|QAeD0|1r&1PlVuBp=VK#;iQQWM}1FY9j-(28x2tC4F*@e7Zv^DBUUE>?4 ziPi;77a(`Qj7db3sKiJ`)A88#_42O}$C4||<$|Qi)@f+5Z2QI2A!m;ht%AhUCpq*<4sFad zOb)dR{3JpC3MEL70oT%m4Sx&0AQjp4qBZ>r{`*973gD z*}v#%RlYakJKts)3FM}``5IT~CJNzGEJg<##FM$+8{kSO9}G)7>0i|#muzjVTElj# zaah0gpd-&<1!xJhEYMU4OL!N6=M@|UoKRI<<#Nt$cnH2lW2&{U9W00b?|Z;Nod@AQ z5DQNl2Vqwe(1-Gr7B7juVy3#ofNw^HJL{RDommfQ>&2-zQCp$VBA;$uv@U8I({iJh zceNmCvgaYp8E()=v3@C2iY~`5n)%MI)T&IWnC6VePN%sTuL@De%mY($uyqWmN9cSLEm+m0p!&b_VnR|Ht++533EB?yDwg7A)i6`(_S>R&CiV zygZgd`Y)E7bryIqH0j(`b|QVL^fx^VYY z>)3cL%)KkPdZ0r{1qjtBdjpC068*Xvx4qB(eYY7E-q31{2;G3>|y;-1;edFFSgTz z%iPdhD{x7?5CQ>Z`0PL17A7dHi{Rys$H6Yd?4y9r7L0kBR`6d1p=-m2G$vZd_r=$tyI&oM~ubF== zp$^_}UgK6k!#{Q@1INf@>3yN+k4SNVp|rUe!&1*p^|?!wq6p#V!Z6Ep&T;_0h;o`~ zITAiHOwr|10y=2B2=bjs)`mfG;n!|9MTHgla`uORpFM{YFW5cHXaDwZE$vh&sStjZ z$T>~iV3V~6>SD%f;i>T>6Dw(8BB3x5;$E3H5T~l8-^&GA%aYtl0+sqM*^rs44gthS2W?#hSp>6@X|zLMuj!oM#Cme|A_36Si2e zsQ3%2RTZiNgeP0adWnVDk;dR<`XGx7$~j?tEc z^*A-z`7ih6c!8$jBMR3M?rF}*dwH#QqDv1 z#lEQ!p>XOoM5v0OeXOV6rMqS8o^>GD-%eh2cVdHl3kHslgHZqjk*^69Q2INlt#7BOt1!Q9oM11CPaCbgu zT+pr4Ab^4kF*0K`hsW$vb>tT^hs52}SfVC+$D`tGM*cr6m%TAKf-PymtME1}#uSJ> zL8v0n$tg5Pc$%^pm5(#4xna_G7cKPP;rI*(8^F+ZC6XUZ)evvd(DdD)23uOG4-l^t zer!_l0?!#^d8DJ2e`!*qBxg%FQFvCiJWl}vbDI^rkeN*6a($iUjK*4DEpj;TBP6Cw z#wW_PL=-dyVT`7&Z6&B7NuvsV0Z^!GTZB7^Mqc|k2%Ze?I6fkz#$=#)w&!dE6F?#H z3p*OCH*~FZ*M2imrd<=6lJCTa{`6!0%JquoT}OV9J=Z3&>#3{lj|+G#Bw0ZH5-Wjb zGVYN$v}Epyy~t`CV4)a5TM83%9W%@)V-3|!cht>tVZ(Ud00nXc9?O*SP78L3t#=GC z>|r2b_W=eQ5DzoX&+vfIuiJvYhFkW7fL;#+@j3?Fb%d=j9ME-qbi;u`Tn`1`I%4%g zAzR0<-H~8gj}Fj!Ola0opsc}gtm6Yi4v#PzT)1Zda5WHWHPDPE<-&~5`DR*{alXU#q;QB!MS|h_PWIpF{%mlbS^Z@k1 zgnGb@jnwpC)0VnPx$$Kc`QRWonrfH9#^`SH8<*3F&LKC{<`nJ5=#}bhZEj1)#+o^a zGJ)(Z1PKM51?^aHsErwTsgHS4-{5xV7d|e6hs;^g8o#4QUSFw8aV6uMnmXQt>PSxQ zf6Se^?yPGC2PA=j`(gOv+v3to>F4WPcE3B%fo2+Id}6lD87vweBREzTEm*KSrnfid z%J}Phcr&HO3ZZ_U^BRBu-830_ndx48 zZ~gMR$;_A#sUlyVe|i3*`#@cqAi;<#Hfh-zJ*F4kbNhwpnY=Ln`=$BcugEVwUyv(u zRr_E4KRsVe^`>}B^kPzHq!AU!0X5?`G!EA(4h48oBvS@U@#X*QzOeCIL>3@Ym=<1~ z!qm>lJ(wv|dW*KO3&=y$gF5|c`r?d8QG30!n>#M`g=lAV`l6E;ZchEnmZ{ff4Wp)F z4MQ2vCoGG&5bJxoc8tQxCrf_kid1Gfi&)&tCcy6w=bqb$H|%WPi>P>6oFG23!O?{) z$@%nL)pGIzj>vQC-5IF(=Xp@R`f_UzO6t!0&R2nJ$#c=u7OvK0Km)rvz`cN(I7aE% z@~shpE6!HUwA-?9C1Ya60Al0-@OLTltjs#~39W zY>^spRAEIL;}noaaTp}B%SCkp1y3e6-_ST?$egX{9b}pV@A9bJ;t%NSZOFmz#g3Tc zsOsDFeBc;VW0VDs^T#H)bJnk#bkrv+Z*!A>IcY3{`TgJbdBK0#eZlYLhQ|Nx71Ao| zZ6(@&Kdn~b3?eBuIE0q#JZE_i2^yJ2@uCFoF(HE8!-ZH|%Epp;>YcqWyARNuMYL4R z7SxQ1(%Wj)+OJN(B2)7GOq004GXz`fOCKi6AB5nQmhbht_Z(SnLLVFQ!C~(+A!By2 zSa89M^_5I`v~v*E`;8X8`)wuf$vwk)!d~l@gUT4yN9Xz$4 zzRNHQk>)+&y;JgEQpo%zN|FLV&)vn*IXuIlDop;ywbsK&r{oee`e(nR z1yW3(ePNN%Z6iF8+;1SFVt>T$x7SPRx^o)~)+FD~9`;4nNI9B4%Caz{m;ijz(lAZ^ z6N}A@3u$e7O}kZRS6J4k3ILka=rr^o*aZq8{X>vB4&_%*=ayo=**z%srfSMsUTzuO zY_MHu#hk(HjE<+}KUY!MT*zGYy1%q2#A7B39?`@X{wgRpV^cKj4z71Q%N|j%YJo5s ztuT%oJNOHh;dwSMqgz&3$4S1{9c5)f<}{%q(kB8Wn{La|!rGe(iJf4Jy`#d^2ttjp zghTOSL%C3e?j3_hhbMZGC^^ym#x}fR`NA&Df1|(2+;WUnnP|=tBp021ZOROoX3^k< z-GgPbB{;Slr1h*`6J#Y-A!y2;O>k8;h2fbqlaQv~c);q2c%ntMW{2*}de2%=v%UUo z@a4AXwcU=f?UCLv*6ez)H}#?V8Od2(Mnf1jsz((d*iWJMIom0KKm`q0xa5Joq3k@ug$C=DxNL;OV1YWzBD84 z0c-QD;Sfdz@P+p>k4-4J*<6@`DvTP=Q+r8I-~hVQ+^sf(H4JR{ydYnsGG||S%^ZiB z_XP}1tzEA5Z=PJ&`uB}XqUEh^U*Pmt}5Xp%KHtiz06q z?mf7PFWBddRc*@xoMfIe3dPOtX$uGH4^@p4 z&O~w32}V>~p;z#-bF8q>naoX^SxzISX5_le4AO8f0JRp9Kdp))`$S|;J}vVs`qWOA z-Qn<4sLIyyC*5MZj?qu#kxXrXQ^eXmm!hm@{V^Gyy3#w1<9Ya`23fb zf7YwenuH3ztZ$iE7OR(}y^`L*1Xqm4tSeV!Oz7XB>|0x`Z(L{-MC}>95KaED}ctaScSYaiLgLc9~*o6S!QEm@BXi*t-pw0ho$P1(}c&L&+kMw7`J5=cHKWvRtmZoJ@}lI(G}e+=8b}mLZJgor&&o z*_wFK*n;5q)PM}2*@70G;f(m*;DSE?#jeBIx>}R7d+)zv@4r{xf2ZDmEB|+O_Sdua zr5j4|Rq?L?9v%7(!l>XTwNEt;bYx|rlRWfz{)t~xy zKKCmI+iD;I4R@0{;YG+wY9?X>PE5_6z{z#Xch$469HdfavsSLiJ5FJbAhz z1V12L03NVdLZvErajJDv4ASUFpng&-hvxC5R(3s4mwR(To-X(QE|>e0TKS|_KB<+> zcb{&ar<>>L=4pAJ)XK-NR@+_Lp<3ytVYk7Au~+4_1wpcxQm+U7N^8V`;=Ow<_^a}Ffp zY$4tRa@%3CCRC&uC(djm${b6KIg$voo%nJH(dD7UmPakJ1RkI;QDwZC0z$|zTL=+R z!o-td(WGB22@*-R5l1j0sK3NiZGFEtvBwxx1j}nvuJ(AD_0G3@F{dtkcH>VyEt={- z+eD`ub^s-tDic1|w~ASo$hAf7bBK#jQVc6B3XHnttu=oONtR4hg$BI#uM1je#&pdJ z@_QE(kBBJx&h7|-_Pe5j~ z`jD>0j!)t=5)z4kF;f> zwk@pW4b{V?qa@Y8Kvkm!q2?{P$U9KfXf=aviXK6CS}fl+uMtFh9H-rHBcjv7%V{@7 zudCt%DT3Y)iJJQ0nI_4}%ag71IdQ(Z%;=JK$7Y95yV+ia+>hy;sW}Ka1YW~iw(`AY z{oPz%`m1R26HJO0T@#wiQVgLIx&kck8UuZAGHlCM24SDNR0~j*T#y&D7ehcmAGF4* z_#OMeXbg4Zs#Lkl#g$EFl#l`XMwqbt<{+K114awAB^~$e?BCSGiM?V(`twe_w_*5- zp4+{lQ*nhuwuK&;()bb~(#}XZfwx*0b9O6UwW1>K%iMJ1P1XY_8UCq&MVj~p zm<+mE$|zVB^wGBm!P|ooUEua23-J%M&0VeK#lW;JhF41CZdK8nZ?$Z*3I?RvPj&Dc zJ&o^WVzT0O<1XpUKz>wAGuL-%%BmrCcG4WGV@Vt1Y#)ysPG=9p!>pI%Vg^|YKv)nz ztDx|GUp*|X?dQ47osd+?GKF+d1IDsJl@uXxF5Ua6*}^dQ#?Lyt0p@-x^9Fs}qG2tJ ze8i9&61#!mtpLyTf>n4la^Vic){I(}^Fs(!Msfntp^~Q@5uK=R*(!)vKYK<_v4s;w zaw)sPqC;2U&F5_A0!(i{XB1@o>Zvt?BI}bgsv@A-RBx}Mx z12WRxwqnUZPQ6twTK_8AfgrX_l*eV+L1hirR!vW~KAJ3xtZWnFeJcZ4!k~ef>#nNE z->J~Hv}${Q0A;ma(@1Y$Y!4Zv@nbd`@B&4e?+lR&t~K@#d79asuT;BS1@e&ftEFEdO%-q$D<0flHfL3 z$$Jt@a?ePXvpaoVf>i2^a^BLy4GvHfGUZ-zZ#W5kUegd42w=PQ*?GuFLUavUTJ(Mp zyHerhf=Da}F)_osO*tx!ti%V5C$&^u&I!d z)M4ARrFrwxpeigR-VLmJMX{nV_&as`cV7p`JvDg9Xr^5$>XrlDSPdYBdMWm-e7&Z%3VUXi`W*GYdabEc)1 zAOXp}-4uQF@8mTtC`v%co?2aTW?c5V)Y8oy>gU2I!?I0?yb-~Lv3w`vaG~SG`^fV1 z|I#CTg};+jw+2Rs{yL8z7>&tyV3vaa{4wVRo3*){++u{>UYf%DUCWIL*$?bw{6H26 zw1zW-7gg_s=82pWL+ksP#rGdBM@{1U50`ch`ak7jWB4${-Z|`P_=gf?-!`PnFcHB6 z-)VfmAJKGn1feu&*>Ab%4MeRUWc{{QmRM9lw}Eziy`q|zlC(?;p0y(LRS!_TrZK37 zVhKp=tiY(qYli02_@%8_IY@*+v8|20!!qrL!MdvLn$Cw?JVl$caG#mL0SqEjtT>z1 z?4}BqVP}SPFnMXiX+%!--_F4qbvCP<84$i!iB!F<>qGEi2XQMz{29KjEl=)!;rkDl zJKg4eF}RzzxqiF#Hiti4v;2-n#$4LDJjVKUxC{MYv=;wbY0c(ajS>l}Pwa&XiX&9k zHZi|`N&4PNB9R!oBDpY}4BFNt&<>AUpWy(50X{Wue)bDHJ!GfwGwC^ZfkPTT-_-(l zzhcL@?HP|)wpxVMKGUzhy{?;TU$JaO-d;Nqy*jyi@B24Hp<(w#k%TW;!8<1ra;8%$ zcpFmWoYzUz}_}0rtR1--Fg2&wAXMWJ4AMPaRjLQVLB>%qvxu zFP+8Uz3Nx0a%uvQsj(TBNrL-V)83@m27GY@hHWdsn)W%Y!d&Xr_5c7It_J_Sg*U?eedWmwkP--0N*KuMd-V{cu^= zk2-;Aq+m0cOuStxv1Z}>VurdceTJkj_T=_*$!RPVzN2@q#^z|8_LN8`kLL;*R)z*7 z>tRKQi@1ABLbn53sp~-IH{+#r(HYcQEuf0GjmM~mV~h&9Xzok8RzD?E?0uTW)yC|g zMjbP@;&GUzA2?!S*G(l4u7*5Hqt}7gI!xI(lssJB1CUS&D^IPsi!9KG4#;RuQ&zCN zs{3#1Q1owsB;RFBTo*LD?OW<68TLtreUf2+i!$uN!fz}a!V0k_M68PUMUMPfuhPRu z(G23K*3I$`W3Vojx}U)dV{#QTnU%c-9I?nUOkf8w(j1~uAu|KS<26~o15^FWoEO#Y zrdUBTmZuyFj6pbkWYTg#hSc(+X)aLq6huaaEo4mQthi@Pkmt{zKL;`Y`G5S!e-N1y zN@5oAbn``c>)&5=6fm;S|GjwkErhXWg(7oVtW2-^<66UNLjL#lckeAJs>WZz{=-k@YtFL&$aUph6Go>TotA$w5H#L zM+M$?c;`q{AssUmpdY7M40v;5HX#YWWn`hg&SjZRT-viqYlx`OJ!3}|O{u=R&-)G< z9dz1^IWW*zLp?DykZ)@h*KF7m1l*ypQx}w?0icx%<VM>_<#gyi^EOz{J z2HAkhCR7fndutThKH#tejqMdOG99>LTLu<{xtf}L_PcTStjUl~2G$*)BVVuhJj5l7 zPdmtuIkdibW4UNQfJXHk_U^>zU^gf}q4y|0VYewhLlFk}j6WbY76BK)Fh=Bk3%9Uf z;{*}hGJ&9TVOT334LVmcDGO|T%;!F5hJmq!L4NTNJ8m}QadNm9k?n@H63oi>1ra-i zx%S_cVM_Bs17C)$VPTfAF=RX50(Knh*HJ8A+gQE!Z}EB8z~eAf!6~{1r3)f0 zkT8P}_3FASCk3DP)%XoQh*mZ9uiuNT_IpH=WI*|q07$O2mB-L<+mw6D{-D4?K(?fN}&{e)6|I`CFg-t^eKptp^Lysh}vS zhb!1Xr#1pdh|n^SY_Ow*Y#qfHMUnG)S@biy3O?&9kTC{s^?1*AHDHj8nKJj^)U1Ex z@$L#`t1i6^U5(h{%*l-)GUAUXio`N1QznXPORFF`G+PdDPA_o>B&GJrTZ28s@?L0; zesR^A)nCu(uip1n7yC2Oztg!a3!69EbA4kbTmkA~jj33W(Ei04>)3o<&KRs|Bq)AwJ~oCUR!IVuy-WET!=8Y z=6fS-$ibXhncPVHHk!bM5`xpYd|FPDtc>@vWqcYE#D3Y}&5baqWa+4my>8YOj znW|iyVEuOYE$~s3$J!r9Sv}5+tiNUAj@;3_wa(Y(Gg1VK`5jlX*IQXVRII-XphW^Q z#26k{|0a0wtwkQWDOQIF_J;8q(hgjHi%;$dzafvl7_Kx=kTVx77ca^GJN@|kU#4e& zJw5&V^XY&7!?A3?*bNn>0TQ<+b3>0biIZ$Kk)g4r!%v%Zi0Hi)f{IvsmEYfdi zeA=%d`fP9n;OKhIcg~i#?bI`q3YKOvr}>)1#^SZM*4I{4u!GelfZ@fwi!lbljE1xcpf!KB?xA9UsYGDIZOm)7{0xn5Q6+NAIb;p2;lDq7W&$8mMuoE?L!gU{*@<%R zVzyWz>N7El46GnmF>R2~9v)Z|pe^aSm$o(mkS-2`1D;VblzdpsW!nwLk+wG<5?- z8$R>2Oi8Kq2&JfhUk&iyR%8M|I4M#nP)K8R*UXBRil;A1Y)XdM|WI$kUIgWVxHL3-F!|37OOvR=J8TVMnT+kE-*ypOs_kD_T@4qO3+NLB$TX=XhtJm^yVWJIgM_i)Mn1i2)l<9otXp+f1~TE zg2gJA<#GktZ}?V}P=(2f-45v1c8waZNW2y_<&mqilFK`;xHOI;=&x^zlw7&e8R)fo zl~A6NQ^gp0W9c3La*av`nl<~Qr~J@D=_)@91Gs9*g_K6^g?#u zJE^_b3~BVu@&Y(OL(bzJ@MYC`yo&FsHiGxokINeBoygxGL?Ru4H_jv1n#WH+~w__z7X6$k#J+B_XDZ$9RNb%sKm)b-BQ{ zI9qZ=3nt(?@2HJgJthqung#|zVNFiK42lpd6H-VCY^s-Y>gPfb_L(cd@4}^pouMY9 z!3sRON4?n&%dIa|i-6TXNyryM7GF5(wh@dV*+@d~B)8>JH46~nXBdDoFk3cXla#B1 z-m)3F)&h?oGl3f^2$W@FF)2mCyK`iSQ7c^kune9s1mFc_q_6ZWXHzS7;adad0#g9a zu0XYR6GzJ{iaLE3L$fB9IpEZ7qzdS{OA%GF6>O}Uk@p}rCH+*3(Mo8tFzzV9BBm70 z*BI2z3mcAaVx6`4*uMZ&V6oYpMN+6DFC*9~O6Iw|WunTB=_k^9*m)Hps|5xjNwlJ3 z$sCRZ*6Nylu3< z?%wu>ypqKvE`yBM6|W78B@eJjYguvMi+)ofmqhi*?p7go1|Kj{@SJ(zDudmay($GB zj{2_Qwk}zrs@PDx@d8>dj0A)u1NqACq%o?rPGqoMqyr_*&8F|~p^Y5ly~Ogv?_?b2 z>KyMYu60o*7-6280)rjC0AUfTh6safcSXCaao3+Mtw)uZJattJVMtRL()2|@=K&#; z_8IA^4D+biXUn{P`)Gc89qgCeh8n6K%++N=V_e{rNq%NrbHlI|{h}tX8S&j&JTSHG z)S5tMX4@^^Uj6m6Ejk1_vt2_EYVFd*i&{NK>kc{iB5W3Nm0twQWlSIqxF;wGh~CGr zU;%42bIO%q_fC|g3IU*PR0ml6Y|v&~pJ7MzM{8TV5Nlkh*B(KjQHw0~aJ1qj=seNV zgGV%OERE~8kltG#*dlBR!0hLL7S=xA-?a~CRh+I-3Uj^6sN1g}USTwnE!o@Gc*v3+W@^vkGOYs({@Wq6PZz!C8tv!VaupxwnEY+y9Y zZqeMN&IJ!Ir)XBI>hWPHQh;(`42aUtzH<^Rm@RZm6T%nzf|X(lKFXjWRE{DeDbGwe zGnyNKi+CgPny(lWITeFBH~0WU!}iWFqP)X>Fb_imB-L&2n<+ddYefw%V$H?jIs*fS zDx(Nerx!kd2@|esjN2?vN{eX46j2h8npHuQgvHKrgc~P8>M~BHpuRkTEo?bFgN?C! zW(hTxsYNLwjNIhK+AmRE6PCdcp}_DKCgLs7f#4Yx9Ff2+x5j%->sXazz`&F0tIA;& z1k#X)XD~r2$rotEi65T8#Wo_FKc7=?fi^_BGbh3+=4!C2Fn&f%^aU*onbLwsV6Wo~ z&vTZ#f?O`d0`GK?Cd?X-I413a-Kz7NzNTf(V)b5nym6w> z-#tqb{l9%2GAQq*b#8nqFb$t8YXLNQHK%bdv&>la)3fd72JsvD9a9kcWAl0R)-0J| zIW54eXh3-3+G-9Y#L|Y(NF8WkqtVUVzOnWl^Vh)fae)BRKrO#}Y|mccrU4Sz8Tnq} zRyeNi@}Yt>Tm#~QN^W4s7fH7AdiM22mwZ}Zm)e+Kf! z>XM}5Du+Is?|rGK&d)a1XZ*Ygc`pm&t_NY1cG}k{Fui-ao(WbM+$d`z{$Tt4I@dyu z(JH(%We>824D)&^B;HI_^Z@*RqKv;<{~a=JNv(9jj9i)5u2i~> zcZG0^Ja(rwsCQM+qEvNYj|e=_NA^g%eLa96hRFb-tbROnS?%laL{PG2GAP+DA(TAU z?11{ShullD!-f4VCXV3ebHiqAo=cUR6Uj6y3+AyS!xX}=sgY-=h6{L}QbCt&>bjp+ zDK6)uZmD3KHgWmqPY3DgAPqks47D_mlDWtGj?!3F&fMpwKdy*Oc~LMMC+m*Jgz%!4 zu5J!H*x@nam3?-Fg)8AXco?9l-gAXwu^>Fnu($^t+Nntqv-W2<;s|2z2>T3P0UXqC zmXC~j9dq8pKp1L-YCkksadgfS@>9w3HPJ$!O&16g(4a7{;xXnxSbxqTEJonSdGlEd zz5uJVm3G=}qz;z{OA$<4ZW(>C7DE92UxFZ)?Q`+|b@E{g0_XEeh4Z_EwjNm;%G2(c)rNLuxbWc+59+mYYcAmr-VF&x=GV=RF93|CyMzilZh_>e($@KbwpI7$Y_2_v}(UtH35}49p2& z;6Z-|e|y$0nvN4W52?@W8X9N}sIL-=9*jeo;hX)m71_G>-l;rACfV^pY{-DQ5R}d` z>JT@}NF03iJ3I6;F&zr8UhN;+KJb!Hxf-(}n`OH_fB`?FLgprcbt5JeDt=XUH->BU zRZkT&kTaD@jGb(qppjZuP`unmW)&nW^GqsJ-l<8*!I>i_ZO7)(VT!1uY<(Uamg_|W zRSTUq9#W8y?Gp$Ziw*KaWyk@=;4m;j_XJ}yJhzlqB?>=sr;znALdGyc7X^Xz5~K^S zXuadg?LlIIi8Pm`FeGHrly+YTJu{n9?YcI^=Oq&Hg`S^Dh7O%yEno(m)w!ugwQfI^ zPz#Bm^-hcBxaRFk5>oKqJ|>=#k}HNMKozuL6Rb73NW5wjjHEK=3locV?fLU2m;-Op z*nK$LpdpM|x}9*XrWL1{0t;t>w#R0)Si8HwojLC2<=^7|7^*X4pY1OU01&shQ8kFd zD$;(3d-?}pYjQ!}R&^J-0vk8Cs8aeMa1kY?aYG}Q(mdlptGd-STzzWg8XKeU;!{E` z!)*)@g32952Cwjl*$2v;A!_9mcBkisfW8J_S#;*p*Y;%*)y-gU`8;Rj)rtzi5)TU( zeKnn9rbR)c74mILG#^Tsz9Uys6(pt6iVFuDC>)|OZf0krJ&=JR_SS=xaW%y2*FmjS zckOM%VS;XaUlk`aOac}T)iSt>;NAY#yX(r5%hb8pxbSXRW z&meL}3xN>bt*kKV%@9(FBAI5nOu36PHdHZq32*M48T6-Us)*-$G0WD|1+z<3No<^1*%uDHtFzE##2If?>1Wtm?by=*4;%S)_R4`ep{u0%{itQV|?GyGHkKV4ZtW`^O!2l9@Drn4L$)L(; zjL)5v;IPQpXBHV?h%70Wyc1FA0HyA`YuQ)Dx=Y1+K^Td}^BTqoe7Mr{zfPIuroHTZ z%rYiosDAafB2`?IotNKKZF=}ByC=4ORP74$)a{i_Vnx{JBJVF?8p$-(?9#QjAXKb- z>-Pm|I>!-R;$N@mf<^5I@=-@&*V_i`OH})L1F5jJ7XOg4tC&z6>QMQ}!-a$b%?<+* z*tI!$ss81iQtFeIzB&OnjtCnKZIGg@e zF*EIXDKrWwB;m`D{!8*~sJzcu>(zVjrnA{a0af^GkP8ed+VMLEkdqdXQHx-4iMBnP z5FD`L4OoMcVza~Tq3 zhzjsU9Gfgj1Ja2A0yD3c=>#%|CZy!?Uk6(qY0s5+EWhLI{@f&%P4!YtO{(%aC@<%~ zgZ~Mf&+A-f%u8OZ%K0pksW<$o z;!AbzwSCT&Dw#U}>QDdq$1o7dK=FH!QEcN3K6I`31>om~6&W(`Hmr8$M!Yqwbr9;9W&D$l@cu5NeLhBq3lpS!PLy&>k;x;k7p zh<7$W&QRW8Lubc2H_M1suHBfc+ec`5Z)%{n-h@{&W-Uu{Z@|CJ6AD%F@fRP6zck1s3OmXYTIA)lqWP#*0pllRU7(3w4FgT z;qzzYROZlYL*`uN6{knLvc%QNhw&q>8wR#<|t80i0eK*a>_vZF6 z*D~(7UI6>XV_Ef}gMBcgQ^S0;vx$goI^7=GO81+uh7}&9#I5BtUVLDg+*vxqW*UwP zUg#gB;!sbO7K9>yipqp)y$RhB-TU$S&DDoDuP$!hynac(Ha&1(RG*O>zeJFi?P}~1 zZVVH{zkBS2u3#gS4WNQt3SLyzyD{-Ck-|h)=~j*VWXTJI;f$T=Kik$iyQ_*5KxJ7s z@}2xHw%Y36R5Vu&*Frd<Z_T|{ zbr@QwsZM<}LcX*4$$;0|2=aCDyH&wVgcGaN@PK6!?4;$~AFPQ7Uh+Ew1G%$nq#91} zF8ebu+U19DcOM5HyAh)>=hTcLl4*){XI2id&27uTbuq*`g61&HT&soKn!#eZpGD`X z4Q8>np@!n?em0l_qSA`?R>9iT*isfV%iMP7ZcOJN-dwzX_hy!M?OO==tZk(S?IO$aOXQ zyXrX>p&M`8sq@?{XH!=oX2BCk2C6Q7j|F!&Yz>UoN{L@5>!zWx5)08Az?A{=?W=9) zN)H+v4_#t`m)y9?vYg-Xge{>ADi=#t;jkH(UuV;hcg3nJ#`!AVfdj89H4X>l{dYI$ zmMT`fhtzk}x%5DKh=q{pw0nndzI*-M>A$p8A17zzM`p$gSwuz7qk`P8q$}78bFOw8 z&NoKEnYYgMn?m+%JlxW?b~sa*+G67=c2NQLei>L7L>qYz+SAaig;xqCYMmxr+)i9^ zqJ=c*nT+T^SfbAvO}4vi_DVK^7frE83p-f3^~wD&%XrYx5qwJw#3($9B@3`=?7;fD ziWJ|K??WRsrZ33l^{ea4$LwV4y6a&wdy&wi70oWnV)dFUzz+=WW;c! ze?^W!E_lIdA@hCuU}}vzqf)5~nc^t}3og6?!6k#z%0`1#zBg!aurFIbjnsw- zR9M7*xqYX%$3GimD%4j={_?U`oV_YhXsgv43t1$EV9`*`->_ZzDV)9WpG3n3j8pwT zB~5UNhL$*NNVO&28>Q4Y&dkH@5{4cDgDDjjNWsY!ITcE8Pq>TbEVWH>5qgT!FCnX3jIsx$X0(cLjeV-*Q!8!UDbw5we)gq$Rj3 ztpoGypEQp1mv|c+9YyB-G0j}LLL5}M$bi#gotg4cH?OXs6-EB^$LAo@ef8xZpZD(n zuOi{;x9jUYRQ5nE4*&pUv>bPh;q#h{8{Pth+dCaiMea@tmjs`J-=BitpMu|id%^EE z3mfmcy_uZ6HUZLP^IF9l_3H88sCzL!@t?Yw>r_ivoqzKea$Dn7A4?A1R6i1JC>gfO zDSiF^`k&uk{O!%Ru+k=|O&?~XDeg1kd1t0ExX)sDf4gXS+68(4_Ijj~{ZJkcvFKWv z2XPU2^#f{F7`tHQ`&=l2B#E0uuxq2)^-Pi%lN^Y#_FK4tuk~*0tk(465GM0f%x+E)@M`Z#l6hO;8xW? zX@tj_zcrKKJD1?sb^Na3_%$lDb{)gVbPA7u`VMxL{)*v<@KB84O7;U|l+k-v@gxdn zIq$R`n$OtLY|U$O`7Z$Q4r4R+ZFH7lO~WM7yCQ{OHPHYHbtBwk<-e3>3QKCl@?PB2 zJSG=cUC}r}8COV%L-^LCa;~Z-AtcwJoHH`#g{oCQ@C*c+Z@NeU%}g}oFf9d&0)oh> zpouNJ%!+%)1bP1aIT~M||Hps)2Z553F^hP*`69ga?=N2TclUoU-hFF|qANs*#9m=f z{D#$p{O{}U-dAwu^}hlsxMdlTmAyb)-=s;46|6AS4**w^S2MyefPnxdz}n)=6)YhK z_rxNBs9~2n#}!_gFv8@Fo4i!T-%1fD0Vwa|k@H_|^nyN5W5x!vOB6AMyYy)WK=wuT z_9t{T2J|EsZJil=zdkio5Ly;;3I$bcQpN1w{w<(wsDdzoN(N3?+U(5Re!uoXSQkBG z*T>Iz!{)YjBaTDRlub0h#4!bQ(?Ho(d?{Ft))}a)*{IPOHui5&meyB=KJ<#v zn-wE6j~Nz>S%7=qHRxR;kp2VZ1LwTRX>O{ZnspE8NOK}Av;*xypIE22KTXp<;|iVv z4PB2YEWB+2H|zt1waxRREE6uG72dWD+9m|0cN$u>4I@Dyu^Y~fi{NC%Ov^Y$Nt0wf zWuG-~)x;0?Gvs62+CD6}t|Sm`ma_t(!=B+hgoI?OXV@SV>M08h37NEi5<0g#mak3Q zL7$ATfxB((+Z&8i;K=qjS zRFQdy(yZH6W!|+u@KGx>AE!3+4tuaoedgVF-~nnhKX8@ihpW@PL#5^|Yc=mcj@#F3 z-dn}yZE7};t=ha*-R7+-H*dF~ds)gY>NoFN!Ffx{+n$|0U>)ZNt>pahwVby&4WmEU zt)g>S4gx;AsiBs#T5LJPtT_ z*@VPWoD|0NlF1l>+{(;GeY>EoxMihJ2g*)W56zkGhTc)0(0O8Ybci!T!y4qv+vugX z&O%Z=OlKz1LrCBb0PNy-SZ%1+ZQWQG*i#>A0B@{PPSB+AGt0YiF5U&DriGpo702fHEPBMk_6fP0`t`1w|_nzSi?a2~9R{ zpHRwm%Wvt`3plVI4I9OzOq`C94@DJ18`zsQq?zMpO;j*5(5oAmq*|+BvHnbrDxXf; zW>G;IXxcYH5T)4Y0}3~W_XD;e7(BIRHxD#n#Up@z(v6yT?e4jy4>c2bTcMa1IbgT| zXQAi8lrkm#Q<~qh*oj3ma-}D1Z3%(B<@UOQ*fhGX0odxHoXln?1|Vw_&Nir%QHF!s znR-Vvh2{<|pMY^w0=9CoGOnp`WilvayLz@U>5eq#cA0X`JPmD+G1F|N@uq<#>syCC z(%=Vy8?|C`K>+96U6r`5Y`% zsQHA34mF>!P@?7&u5945%OUikZ}XbAV#Eu${6RO%Jccy#?3t?sj~|^YnUqBcvTj#H z{+$GawWh7`i>TS8cbkFafY;2}_O@WUW39?#EXsj0KR#R9&B86nJ6n&(*v&!_JnZ}1 z9HH&dJn%UlrUyPpL-w%UZHMn+7Uv)+ANV{R)`vg0M~nD@&+)9W_ET3B%0frh^rgVH zl{wM3J1BP6jf#+zVaoA1v*0L(xr@)hhV5TmU5+%hcwcrHTf8@ZwdR&fw4!)nSLAgy ztpG6L>Is%Xw^w90hW1vASXInPu(;7EkBylSmCzPgZrgvVSk|x@XR@Vo4DAoVi#4Bm zf%*?tYBI)Tv{QwT8Xsw4d*YYqh@?Wu0t0;}=&I<+u4#MFQR8Kdi&$Sy7^!Q+=X>kl zH=k|e$*N0orzjrNhFvNZtyoG&`h@X$tGDOM=o==;8VqABPqhFtK6J)Uh99FLW8}AYId*jhBF^^&Kk-p^y(QHoyUm}#!6;Q zWHPV&UruJTlQF^rj&tU$-(|6w{)sRV$ruVr!npI1gVdsLW%lrGhN-Q32-dVwjiMh3 z<~L@7TisPQgZk~RV(3VAUWux%mWEkIS3}6ip;eA|!c%Sn!8PAwYE zX>69VROai6Yf%5O)`NND48+!`%yTA+WbFyCN049Ll7vU2R8RffSM^N6g6Jfb(NfmJ zm9$oq%5tH9W#W$9(Oiu?oI8l1F~8$#e2xSa>+eR#f4RYa4lOu4i;t z$g;@F!dyV3x6&LuTYbhIxQWv$MmMRokQZV0=IY64QLtRRB>(U9ci3n;TE?7-P#f!Z4BhtO zSCp5vZeQpL&&UPIS;RALU??d59@rU>r-6Z9YocYz!m(Zl51o5<1tak^1`~R{ z>!G213{3T&D@N2xmPt%<_Ae`ho0;AC{rG`N;T?KYFsnv09#Xc)jd&arVxKdf10`Rx zNQ(HNCcBN!Dj^ztjtFVwP&w~t!ef13+qg44KA~iZr_52Es&b?@K?dTh$yNfUXKvuHM)yyx@^#Xni-8~Uk@k<3s7*z#% z@tmYwltn1QgQtQ1@ed4hiSac}_;06f8cQvf<;x^TX6h@S=0+LI?Z68B zn?flFeDQ#_qz6avQz_@`BCG<5dAE*G_l3}VbdfX1vL z#ky7uG37rx52QIIZ4Z; z;8{{F&%1SJxo?nDw#T_i;3ogS`*AEi~luv-*eU{Ojx*dpQn$iJiC-iDj zi1YWX^CY+-jL0J>cd-~0nwj@h~R!n_>gO{?lhpY32Uh#hLy3Xfe zReXA=CZET39`}L9Qjxy_z}#yFy27}c%cH7%vd=Wl5~gOie**iRIdA9rJU;*M=Hm6c zH?y?sEI+o&f3=d|xI}ANl6#7%mjP7)Z+P~Syb-F*&G&8HXdbke>SVa`FON_uaOY~i z`MjT@qg{3-?@27lJtJAp?zCbF!4`^mr6SBq0{&ej(=3-MSN2}%FLg&kxgAgGa&I&T zjX47a98Uac7=v!x^Y*)yyq^|wS}Hc>!>Y=_YTU9lL}u=3Jxp^oZM2YJGlt)_81vz8 zufJDAmMTv)KW9aB9W{LzcXy59> zlLy|GP)}J%fJAs`#8^A;cHt1m3xqgY2*kF*4~GeRc({OvM;_`h>WUwz&8N)g}2Fo7#B_SuP7y_Fl`B^4{>IW_j@1hjvoF7#K2{E7uM&4J~Bu0kD@j zi-|D`J>H`D8w$a%rzrjbwF1v*Z_~OeO~A5Jxa?eb(?H8Yafm(e;@?bJG_I=5EmSKe z@A~^9HbU>JnZt zHg(Q4w~21AJlDF+^>4!aDOg||@m&z9nW0~eG1YqCSZT*zV%KIR;BQnZ8F4yP#m6Nr zRiYG$K?K@;b|6HDPYmVr775Pouz6#kCb%cDabeeH=gA;l4dL&Mq3o)~=A0cBn+O9J zMPSpw&3@y03`lPjRdYuTcE`DLX9e&y4jzkO^p4KIJBh9PjlTH~tKGd26QMh2WXF0~ z7XD^}5`gh!GAk1dMRp54DllP;cN+WOL3Q`?clSlzeoFB8JZ($xptoxs-82`d?;j^k zdtYhU{dSOA%63h~d;Z5i{9)kg;yKh7{0EjKp0D#8U}R z_Y1;K*%xBUh|G^Lbd>SwF~>kcc@X@QheJI%7}m)#5Kaz^YcdYaWK14eSPt3v19Cu= zk{z*0_J&BZ4;+$URu-1PHjFe9klyz28Eawi1S3HSD!73GiEwoJo-xjrecCdr7F+E% z7N^lx7&%d1S&%~h|JnQZ?#68_Q52uQ^C_^Zd;M&8TXIz;>FKWD%;Y+*O5&S(_?Dey z=4LW6L_!jdNrDZ4ww06f-S5KQ00~NbfU@MOBt+IqSrUoo#(qBx{@knkcBg<$!hpYD zzkq{^Rt}{}jrZ=Nr=Qo zgr!|uxm^y!_iqynP}3@E1N5fvi#6`Jred&&5pY(&8rEk8&GDKw*gQEiMp7=Nkk%gP zF9TiQ_+qJCwGaY_=$ZD zSgn;PN&`j#;gjW#sm*ig?EY=_B4}J<;WWDwcF&dG+#h!&uC&pf<%(OC8zS?4X9#;e zeBFPeE??YU!cW^u7dAx`=85PDSN)I@#wcBc`JE&@TAjZf?t6moIa;DhzZ3I4`I4$0 z3)CB$(FLlw4lgghd~;-g1=MioFB@DHN|J;FynT4)gtaXgL~1wQ+`~5PAUTAiXnmzQ z8xmh!tY_OoFA|rMPdO|SKu^hS8s9u@A^n~u)vUL}v0Wof;4B(`=fAK#rh|0a}nY)_6i0Xcp`myE_F5esto!|CfIOCv8cn&+$OC!aL*`orn#T6ae5 zvOb5E`vtHD7gD55FIl0;fBZXnK7Ial`n0jj_s$;Uy>a_|`s~>h?Lc%UZX2MyOBry- zW<6W>5_F`h=DS~JQUnI8n#>)Wk+z(Y-PFs-A9xPUP}w-8Z?3K`NKPe9nPw90Qe9K% zp9V~Y(U^5b;n(K!+SUZOZ{f|h_8=O^NOxeOlBrx|sw>A}mi0 zLru!D%58a+{UGEbVN@}a(i_7*NiQKyp{!dRW)CxbJmdu9%1sSTr2TqFLS|CjDF9?a z5P={Fcr2`E4(uCpv_QCI zrm6UaCkdSjxj13j!gg}@Ul~fY z*HecP_I%u^bBbApK4<1G3&|IKfyfU#{My3yf9fw;)@Mt>p z3bnVDz5{6V)$2|I_rsE9<{9LOoo{3nhjHw>cbYRBxzwr1FrBhn>o9?cz4X^%Nlc<8 z%@#}@6H25=5+mOfc}#VsGS!kV7EE>x({Fg}Ii-{u5}F}Qv|!P&WAX);l-~2SNJ&WyW=s@D>?=oX>FkvJ!lba-wF* zQl6O=ydY0IcD3CvET;>0#dXr6m8tFM6$UT)LY;Y?X!Cya{Cn=FPSjxG&K%p->&tcd zu7f%3m!2XfJCfMereVu^SMxCM@;QXdm9gx7^%H{SxjA3Fp`F>SH>Zd1IA1pdWv=9h zGqU$5XxCIOSf3B)K3#>cDj#7imAq3I;jj`_L&{SP_m z4_SihBe&fTx@_TPryA<+*WEcQ?1lxT0(Nqn5>w&jrFP|_8(dmq+wxQ)Wxwm&kFHtW zROI8K6LkmJl4?Nu-zgd#1Aqlgl28T6^>o8JlkQ*}DbGlGM#}TMraT*U3j?6vsKUTcuaQ5r~2zfN&Y9?G`VlB3SOneD!Sl970h6jdX>Uel1Ef zZM}0Uo%`77E#(Plf;OLsMO36L>&_QqLpTL52D`*D?q?LtL;k9 z{&rr4sF1hpqDq{-6Bm>uG0#QVx%EfzquU7hSCUs}FTZ(3mNbjilHRam^L1WMrDFgg zV+^2iuPS+6V9Tq4=(kn&%3s-h-;OJwu`2L1>@PLeg2o7V3P7|-~!O~ zaEA+c{-6aQ^MVZK*^cc>XL!5U65=B-i*&LtO#pJC2Z{hX#ktBg`_wV1@c|nTcXjV| zgEdlQ!K!QrWrr1qcTA`v>>dIUfnDo5pCv=K&yW?HG!LpR`}VtIbMW068vE~f_x+v# z9cuZhWXT$T4+tY|__5^&rKv8~0IM6A-k5`wEHa{wn89(xwws24$tWq5Hca+=M&pUd zk_|ShB8F=}bP}2qTfRcCPG*Uiout$Za{{!B&k8N1LYMs$#TOGQqb1iY(uHIvH0P5@ zWVc2}6PcQ(@t;)Aq6y9739h3yk#Dx_gR z0c=Uy=PI(Vx^ggJWdN;;>a|msaDAX~lXSV4Z7c3wi9?%Tv&@$G zdT5P_cN;yf1Nby$n#NSq>7sfDn4F;1lRFCOe#wj)8e8Ndn}UjpTRH8R@Hx3|8s)l6 z0<^%CtjM$HC{3ElUQZFYrWIEC@u%tLwwbQUKaK;Jz!eb%IMvvoAH9UAF-VuZ3ES>& zv}ty7p) zm3XT<{TVaYUNZEi+Hw}%c|SH$ZR2ANOosbOH>+W{%pB5`!Pp3aH*D2#ge{}DB|%PU zzV>WR4bQv{6ZF2qRZc9S*@{q%SaU)5MHVxetjsYYd5!Iss21%H9(m5Bxrr>E5}VI7 zVVaelmqjhg$69u{nc0^+lBsY5% z(1P7e$txNy-M3i6vuVzhs3J|Ntkl!yZHo&AaJhY4Vt`Q6u*weBVm|^3whiFME$yeL zt%U?a=;?rcSzmnfm3)L*!w~Ix*J_hKcWJj|4c4|MyhXMhP_1vC*H7?O_g25(4My1~ z-q)=OLhZ23_RC_J^(DP^DKQm?64N=0_PI0N9M1+|G(IJ}ezcI?_*(r~gec zD|LddV--<9*#z*}mzKAM44T9#F&xKRK5ORVR^%QvXNUK+dJJA)f6(^nKWc??^1uJj zC;u|GXi243Cy|Wdf0HWH2UF7(O_Lpr@$~7_r~mr-=lJj6@ZYD;KZAdt|LxPiJ^j~b z|MKj!fBp1dKY#xG`KO=%%hTsiKYjM}U&zxPjNRJ`rK$Xvr%z*=0UHv>Zu#H%04Fp_sH;CBcwX!NdepG7H)G-R;xqzkWV_cEavynkTI3nMow$ zlT>6}3&TNABO#ekW|W^dAn%jOWb#SnGV+4ZoU?n)G6>KLu1vx8} z7U_G7c_42LXqG;4T2_O8XqE|dNVBM^?!PhhoH3wtH}vh(>9fzLm~6+bWm?U+_AKtg zC)Rz5)YjFw^iK>vGt*f}QRFX3^Fubby`47IYypL5@$-Z(Y@Yg``P+;)vLnLP8O$+MxIo&lW{OrT^9^1k?JzsB*v?O6OuQO&=3NW7Ig(BRc#Ro?} zO=}ujH4D2uU)bTMw|t4&pp6{I7D!1thd3&uWwNwXOKtEk2;4B}>Z~T#>Wv^y2*c zPcLgpFvvUP8)Mkq5^H3F4qwWc-)1Z{U^Hv?|h@LGcBfI>Te#7ob zB_9GFDOfr}Vrc7Ds*TgG4!!C})Oq34&0v9sZmW6v2T=2(sioytQ(z_h2O4^n5SCS|%>?E^g?NX=Lgkwp*y8#Z!WGhLnX{jU zey%P1WfwBE*hFjG%(n3|5E0X2K?sYJ>9I1k$Hnx;!-QF6hQz<@JMLxA@h-#STy`1X z^4M`LA7wmC51@C)uWT8&(ifT1?XBK!;%kmT>9nN{i8xu~P~J08qVJAFsdu<*4h5*E zp%-S1Mi!;^#K@y;&!hZo_s}vaYUENL#-%iY?#QN)kxlX2$fk^J%E+eF*%bL;zCMd*mN-kSLa&{ds*^0W*M0$ zbfL)Il1Iz3Dzaa(wuz*G;tb+vSiD{+(bj8PF>N`Yavi4koI+qGRTJ*i(A|`!T&JC) zp|24w6e9@Ngae9PLGL_6)Q*OTY8eef*iyU;e#^32jY-5EExVeL$k1BU!n>8h4kSF% za_R{I0T z@X+%5z}1icN(@UrRahEA(e|Gk2t+K>#py(cTl@c$OG^{%RM zfo7NPhDL5QH@*D5p<|mW$;pu&Qsd^kyOSAlYdn^G1DN;)OM9f~zG5wp|QDK$*Xe3&r>FLunSJ7+>JT0*t@bN-e2 zfnF8{+Ahn1&li~}zkYSkqM|En!jODO5R!`HFe?|~EP!s7OQ&AalR zf;i$f(dH*wO1`#7pAs0Ucgv zE73UrtB(Gy-_&u52-d-=@mP)t)6w*(t6++uhlOSs%xeyhYsf?a?41jMS%X(IBYDLExsQKEsa!81{gZ96g(QljdsWAM;y^;Mp;zVy;{g#v<08EtPS zejJ?7$ZvHh!)@)$ni-yopKQQw{~H@iO>D8G`jUOT5zr^72Tc!OcMl0slsQZiY`*4pAM8lczDC4^T~g_aWt2^s>* zpfJIaF&ZPG5sOI-Y$S#WP|X%RA}Ny#Msh>K!Uq^YGq7y|@gb?-H(Ohtg5J76RST;b z;aY#~i0a8fv@5veg#UQxW<7A5faCcJ6jC1pvtw0bBeOFyJ0r6b*hXfDjLgo+>;$xt z*(uu}kl7h9vu#UG604IMq#pYLwS(*WFyPEwxNlAI)w@xKadG*7u0YvlogpO67K}Wb zJbU`I@79DQY;&zYr5&~q5S@h8%bM7o?y`A=oOcNPZ3T60`u_cyk1P!+?FltoMT?!M z^teo!CbVleeZ@0R*|E!#iUhz0fIA~qOhjgl1_lO;ttIG2_?)(iaoY6QUzm&Gg8)hI-gg;hNGJ&&$^}|0iICP- zv$>82I|*7KYv3m4y+i3GLyOeu+n0T7C3b(VL@pAsSoueV9wzqgH&e@7c7|55TECI9v|O8%qdZ=d{~NdBXlqrHvHPEQ+|ozONiJE232 z%+A1pM`mYacCd}i&c|Tp2wl&XD+wt`zM=f=(edxRJb6#Ixrnz4c*#ZF5$}S^>IlN7 z+Y6+Es@!yB?LLa8sEI!&zle-scmV{^ij>*lv|Pl8>Zrv}e2kYF<7J?Y@iOg0^u^08 zsX}OUnMB&cXFSwlFxy8QJ!62ZBSd9%CSU9J83@OimEs z1ex__%LbIEx$mHtTDhRX+L1u%<OD5~>3OfUBv&9%JW+ zs=&yD_uVRrxWa^I`&NpP52+aI9NZ2ZtW_E~?p8V|vW*^!Y!80h>7&SQbW&unUWyFT zO_AaHDKcC~MYh_+9c&+PJFmh2Cv^Ax!e+xM|3)j~npv##naAXADHO6TWs;^RH*EFf zI4C~fH6QssInSOTPql%-N(M}1$%;II-<~wHro&jyt+cbjk8iIwB{hinwy|>aX4?k# z-qc2M+^@EU<0}SFc{#~YErA+}=IouDZN`YEHw?@m!W^jLF~j6{Q;yA-;rB2kP%i6Ee_a!EuL)J;8aUpJ9Kwr7*I|nP za=TECCJMOE%usYYHMPEzUAq)CsLR0XZ@>X+cn@i&`BAB7m=ZW|kZ)p<9cY}H!y~SP zFvEJe%a!>HmAQDdHWir7oMuoBFd{*HUpcL`l{)4^&M+Ua7S)tz!UaN9plmQn?JUCv zJWZ51UhgX^OP86u0U6C|T@w!J%oGdcp@9Xn3~7=tsi!@(2u%@|(HST*>sRzV^E4J^ za>fPKo$6TE2C!NKiQnV8>2UQ(=y83Z5kt0O$V})FNw&6+1xaoI^uzGvMt38s41D~N zP-W(-K7RnsJ!<&SCDKhpRo-wAa5Gf#=K_fUCg9yNzjTp#1v9+?5yh;VVuMC=;)uu? zCWU9Et=t;``U{&r1Rzyf&n#XKNve7UHGaxyrpOa#s6SBE6Rm0B>#fX8SB`r4zyHrs zEgM}m=xFH}Eghq!W3+UPmJY9tmX7u|S~`ZjI9fV7+i2;qZM1ZZmX1G;rK1u*_9dJ8 z!k|}CS81wwRE6!ir|_@G7gT)#b9*t>7k4+L{{d2Jt4vx-tsA{RSX}*`C|$J84X8?LAH_K(S-Q(+#t8iTbTGtDQ}6Qu_!=vZAosjx$EqEV&ZvylX) z|9b^J2A48fnVhCFNZy|Zi7?cFlR=VH;~k^wkWM@NpSUT0o9`+W#w z(~aGSG9+uvMrYp*jL5%vRP{glS&`1}@tT!&BdepVSdcDg4O9{g3i=tIk8fYWK6?uE zdDP^2yE1tVFn5LP6dobp%m+7J{gx8Ww3@00X{pG=%v2wF-kC=-OYPev^-<@Yd1MpR zN1K6WAEu`<$;)~hle~nnA>pQ^?afF7;bt2Y^MOo9AINO&S>AOp%Hsv1>@*1FaRX3x8hkQp-(4HE z@BMAmzW29L`#x&lN9}u98@2E2ZPdPx+V@fW{s+*$hX-g3g{#L7%y_Jzj9ms~e2`#_ z->70?K;XqtQ2p=$7auy<;sXX+Y#(It&;W}-2oH144-2aJkY4uz#Y!^Gw=Y=YUxb#b zB`wRLw}QtcFSM=2FractOOInclZ^&(1hq37i<(sJBPeRpb~a;Wld-bNSlMKYrIN9- zNlzQ|eKoW(-&aQ)^L>r^zQ%lCA#Kd}wUfced|zbD_tntGd|xeX%=b0s`+C6RV`Y=7 z{pR!bk4gJm+sBf$f2?fM+kPjRtpv zt+9<+*2XMrW0tis%i5rU$1H21?~Ga2#w=@NmbLaaW?8G+SmR`@anjcQm^{5>kS=tAu>Fo3?_wE#0?9Z3D*U25Ngek*Vi6)Dhi#wsdGtD+*+r@jh{@5r$bR=lviZztM67J~a|PlH+elMbabuiJb4A33``d44~(D;N~0{Hj5T zB;99B8qE1jqC2W0%D8Y7x5OaMiqLx0MGLt?$Nna+7VZV|ph}5_>|v347Wwpskr)*V zu_=bBFNUwfq}<$X2#C1!D0Gu@mm0n7?875Dxps~S(c-RJIRVTv`C9|%ru1r`_oShm zSIpk)Y{53yc(JWNVl|rDtI(x(5blO$Fs_m6I?##CoajuX%oPMW5X4KXfm>;pB0rev zu3x&j%$SP9HG3?z`ca+t%rK=mRrZ1JrG6J+Pmb|vN{6-InsY;N_q4`GCq06EK_4g$ zscF|Bxm}~g?h7AClabp{bfFGmxo5#$@V@t<^*p=v_Jep)B@iB$rmM!%6{+Q?X-;H^ z*cGYvfw{q*84K}yg1on6r;4Yn4tI05yKn-%0pcdG?Sivp*S;Z-+`DT7TL=E;z1X)^ zYd~f-Rs>}|P~OWpnijpJ@2e#|nUQT0kpP%(9J#(hxG!c)){=r+G+#3pV|_Rb${0*^iC+ciZ!=>K^4a=J{R zqpR>UCj{)S+fz0Nckg%^a1p2H%|#=LTASDGDCJ?oubmfBCoS{G8(NOiu6fN%o5kue z7u}*w#SFsB%rq75ScT~J-V4#ZS}z{Bw!u|&tM~9qbE-IZQ|N47A<7!tK`0TgegJ{r zu%)0>k4yw&bB4DQFmmTqL7WBfz`FI^vivvw82XRfZ7)e3!tE_5=CmB4LoMUFMK|*@ zz;(t)Q8N*%IOOE$XJ=-EMx@f0Wh#slQO{t$^CLIwNf>|7G^dIqgJGr~rDXNjpzrzl z7slyJlYU-K@-Fe^$$wtoN_#&!R|63(Vxd-GirS#CJL4t+aj`3Om9U92sF2#A(F`3x z3v0m8<%E2w5JDMdS&@oq*y?(L7(dqw?=KD_-SIyfR_p1@LNq3mOs_bL}w-Ou+q>c%X?8kw{HXj}gS9yOR&28&%| zu5dRIV|FtbDKoj3+N`^z&aJa<6kmy?x8uJ!bP-PBO)z1cWGVe>a*JtaAoy~j>Nfj3wXWi(rSRVhcTTtB4=HwoY~V2g)B4aS+|aJrHz4PHHX`s5gH zyPk`ElgLl^00$ z3WM{*#onI-d+y#qB60E3evA{OOIGSeJq`mWCz=-4g&ehhU$Ys@IjrFzbq-`A!zOw? za-K82A8-8ONpL(Ip=Th4Z2OyPWI7Bu+!86;m_0jk9%UNk&D{nBxTo>7P(gPkt)~d8 zdz~?*599@LQ;t_8jA%X+e27!QQnh>b&i$z#XWYb07aRJnH7ssdnHuY<3=sJ9rF}R`A(onBw^5?#o1mEa_^xPL_<(q>;C`D|7?v=6yT7J)EtrRZdRk z_4+*N^XK(;u?eH`cBnFSP&YNr>l!+%V{D;{xrNN@_FK23JmBGjG*~i)xZwb@?BW1p zHs!Rxb9H-NaPB=ZrT06Ktr7JlH`XLJbfAjQ)g$MhGzOco8@7AhRX4;dXcmsZ7e%Lv zokADv?@^;ca;E^NMc;TAn}_T4;kC6&uiSCHgc5@>ICKkhfWc$7O{$y!ZVY*NZ;lOB zw*Yt^-K%_Kf8f4 zk~Y*~r+IYnSK4kgH~23?LC|Um^CGzgY{ukbq59ZMf58^h_(>%~bbfl5DN31HMC7!bGu6X1>WeG zQ(g(+Dmm-9;KXq=VF{zh!(+g2j1B6!(cTU{(VuMY%nj2nxC2EA2z=E8#Q_*e{zteP z9ki>gO!20~z|w&sXdu`cJJw#jIF!wJgKOt87OSjK=sIra#D5)VwIU4^9-A;Xtf|H% zIO}7Q8~vihCODO5U}42?pJPp!<88p;QRjOMv>Oah3q8+>hGygd(c3trX4OZKw)M;{ z8^jWY$88xPvff}90Z6h<*V8r#FCnN&4nSi?$A%;Fi^jhbt)i&_YGeNUMa$gy6B94? z_9P@=ssiXbVWuR$o%_KO+Qu)V8fc6#p#v>6_fS3i+)m^OnBS@k( zgL)|g-^$dN#b4fso*-bbOhw(3s_*t#;PQDN0U%5ftnuLh*{@omiyP5RihB{U+qhs3 ztv)fh$X>uI>ceO(p1Q366RTaEzi@u!!tIt5uT`!sSlQVqc_fWH3+HqSUZ8V& z!Gmz|f%;d}1`z$wcg=NgB##!3|S*iJucZy{%n@-~} zaHF50@21nK0f5od3K(SFq+M0kIwfVaMqw;Q?t^$k8{Td8M`kge5EIW(%&a+h%Qg{B z#QDG;LoJX(rQCMXG!Y}&`!n7uB3ZzaL=O|vejuqlK%W+eY}_<~Nzn+Epac+W0(wD( zA{?tf1U`{rSv|Bc*XU7_i?il1uacCcO(k!VkVAl9z8v110T71~L!@Z(ZsJ7W!Fl2@ zlx}!w;B`0ouGV~F>n<hSaP^y8HoM%SHvmx`*lYkJ``30!fXZF&i%BN@Bqi^a znAH-W4s>SIn~Xw*51SO-k`RF>jpa$gX~7L?f!{D2mgvc6oCS7-m((-|dP{VLM}X=zbHnk3<>)4}Vpig6Ov`Ee4fxL%7|V_Ld*FvJb``VAP^ z57_!Z3K!yqjdK5r%^9)JTx)l9#3h^mhYftwRnE2DT>H5-AVG>dT;y>UHFTZ#9F6N^Zhh3 zwa3c*BFSGPnwJFFg_UG)oD^Xuj9<*sF{8IMeQululr(9P#$P1HdYSQgG`G=TIGyEq zXO8`jrnP-EuZn2^-+7Biq3x8Pxvx{`s);5uqr=vBBV|fpt|3jStfiCh2fnqU6s_0L z>sh198izlYv(&6Jc8oY9*ba}jCX5>o3BTHjH{2JB#B6j5v>|jNH0Oz({*Yx&p~Z{S zU^UI3jf61a@zNN^Eu z4YTP;M{;DKNds%M#KXZwUY|3SLdN@}Se7VXjLG=Z@kwg5HDu&P-trMdrV^Q3fGx8H zsF+a92~1!jB|lEO*nt^cJB=p$`w(82x#|;`1Z+nZMM~0X&>r*_Os`&M>9FL=yg^Bt z-~z!u)?&8OLv7_T!?vWH18>R*7=-jR4-3D?g4~`1A!D4>!f3O@S##lrrd4thL4%RM z>(k^Wiki5zVnY)I4Rn$snvJC3WT(RmQ zg${dI3nmBNL)@a-*AJ+JXSdDE;jrS+5n;}4u_ZH9Dg6!YHLbi&(DXF;kmd`Xs;j6F zy<$X}fH{_@DRGEdieJEiy@K^a5cTGndbhko$KholZB|`ynf5B5)r(n z@P{;9(~$t?Bq@2}p;h|ebAKlvUtb-S(CtP)Sf61t5K<3*z4goej%*J4xj%R;EuycB zzTMkv?d^Gcy3nun^?AA&M*P`7jiSengmM!Qvpn36gryA zwlkGzC#tksK6;4NI$78CqxNO#`w=(d5JtUY5!$|Pwa9PC#Z?|ZDY;h4rFpYsYeGS{ zpIH*~uQ7?#ZuUkUu9-G}wN=-QBR69)gdv^dn60o7D;U5s(v~BjNDY>kUc=1Sxc=oH zB?Tsw*NIU^YqyUf6O%<7I??u&D{bFb)zO70RMJxLKWO}En1)I8rsMk916$&(S+w_2LsrK0 ztsK%KpOEVS)`_5L5ydOIqi(>f%LA}GJ6R%W;D}5MPe~dJvQ?q`6cJS|Hu=|es^mBz zNf)IPv9AT4W&=^-+{9fc6%r>!*EP+aWipcc8vKyL8Vz&$wY@2CYBC2Hmyt-A4!6)q zN>EuLG`g<@)3JG$f3cDe>cnfpvX>< z`7~Lm38!p+h1EhRuaxJN+AUPKL#fE+j|wF)>zds?H2)q!6_2~B1_0{cU;%cPjICv* zfnfoJIaD$=y#-Yf{A-i4*-;+vuThxz7y0>i`?2`K-^=Usa1P<*wtd3F+*FpFSE1>T z;{E&1K4tYc^d#=EN@WB_>aU37aa7 zjtv-50wnK~5qlI5n|mj&!hlL^W(BrOV6YRCvMl+wF0bHUZ=?v85>1|hXE+%t3$C{$ z8Ybd0S&KK6Su*Fs)GC)W?SwGi$G#~8TFRjMrGF8jA(t!U>u>xhid|H9KynUNLr??P znkpEvg+r!w!orH9m96SGKNtI3ZyS|}LD3k+%VID}xbNTt1T>vK6wmIqCAv4EDdG{4 zVN_T4rBFE9x0WxL7PjxT4I4M*B#D}}9-tkup^)6I?{L^C`2&sLA;Fq^!Zyp>Lf*g+z^Mz?YWRUqUL6${LUb7k)G7^*$0KNc zx?vi`ZI8BAEmSULF*!Agxnxj_%tOR;_}p`JWeT$xuq;&BXitybs3lfK61>JsXR5P< zYyngTBjmi37>*A;iA1_`qRLG9VZFps_U$r|t(YEcKOw849cs_Dm9}q9obg0cc z4NG2Z_~**)8L2)0q@O>$;izk|baY(MhX=vcRAA`+xSC$iTubEfm0lm~Cv*86=NH3L zc(%;@7vXiUAB$Yu9Bp@panw$o@j(ahmGHbZQWkHqN)1uvz?ay4BwUo+cn2fo5!k9I zPzqV1P@E8Z9k855mHw6^YL^~&;H>Zm8s4Srr0J76k03oA^!NIFocDeo?3P;puoG-r z0N>$3q?4`g{~~i*8Gv8QtoZPr!DU?v{z?}-zKOmuqTjuj`oA9!j@gH-x3oz=PcN)- zYBeMeah|C}qvM`2gf02~)@qMx$u`>LOg z7W7$}>)r~TJ`nb>#oX_&x-!aYQXN)&j@^?=$LO#tYDsh$j2;2JYP~_~SIHVa{0e7_ zgs^lOH&jGwU1HTIYojWeBWiComaE_rV(S6XjTxPq7YI2p+`WRLj5lf%ce1|c+O-6< zksP<`Kw`ESWL9s33NQprh9mD~m_^Ny;AGvvlo9OoQT@5_B!|jD->pbivZWWekMniT_xW^sdV4!+zvJ!W za(7Pd`*;zjnXtWEst@mHQtJzKd3bsnC5vDFS54TOXAzl@yzPm;?}aNLxm9puGwlVg zUR|S-MAQr5iO_jS_Vj>|_;kCQ`Y60&>2gwZ^R%OT``xNm?TV3=sYVoCWw9&H)j!`$|`WrQ*2M6`%wkFcXU6bxt0@dPSTBDtJN)XzkP zk}W2~)XZ&vL~bL-$;?k3-H+`@%_~f@E5DR2EhF)LAYHw9Zmi{(9&Z*H60O1FlL|l^ z?eZE19ZU3INQG__A3MW~Aw7F{zoS~k3s+@3G52YDzva~Kzl{+XIovDZ6F8Zm&!)OX zU30s2ufkrP2F_l5eJyEFZ$npu7qF}V zgJ*-LNq9yiaKwLBSK|{nJ{7}*h9N&DS6}v+udaq8a@fgH`p_afX3i6+{YCJxEEKkL zsmks_Jnef)Tt3wohL{}O#?irakbYTe)Oo9OO7XSL4(>S+k|3cjuEA~p&|wQJ+jk@K z>)n1(Bwt~P={0l1Rjs~GjQI_^mi&rd$ol)xMH6+xgTzCl47p^LkO=flOzg-F)cvF8yygJrWeAlgTpa1jKiB@biHRb{O~X zxwq9y5`LA{@Zngi=1HHpZrr% zNieBHNCPyNY~(OZoT8NXAFJsL&Ha6V;4@Vl1H!4Sy(jN;*s89Vk&HGuriBX6;5FdL?%|#B0CU>}8$`-& z)fArs9>BRAEx(izon~=}4k#LWc{Lo)Q!zp&4qi46jx9wgJPMf~@IU6}gqouvRQU%J zP$%B-I9PiuixL(LakLEWoR~UZGARUF%M5sso{(hmhPv>(+ufBQ;{j)9k!ymmxV2Eq z_COnXLF2TbYLNgQRvY%eKay(xrr{JUWbpYN^zSS7wb$(}*Ez>h{2_~?HX>z)WOi%g ziSr`#^VQVIYlrA$pdt|F&uQ@mo{y8h*cN*i<|Mn zFRTMzUzi|aVa|63d1m2| zPy<36ID@2**DLJ}I&b$TlXod$BCD(uEG0lkbUzt>4qr`%gRZPx<_`~8g4WZ z&=Hc`&R%q#o|VbSYM`LZgv@N{i3ylN#8gY9ohId|znlShb7=ZIUUFk~w++|TY`|_q z@5VWV-F^)y0Xx4B8jRHe1w)RewWTYlUFT%|$WycYPgIA!N|jM%M;F(Cp!7k{kL!v^ z;{l$k+8nNP?8+(T{Rh~|kF$=+$6cDgESeZwJAR{v|my*&S z2q>JV^*RmGv|QuQ1dR}`D@G{wC>bqUOjYobar2*#`&}w25t+?EUqPyfv^i?b#!@-{ zUqxez8vx<8@m1LRhl%wl%icPo?HQJhFuf*RLd}g~bj$^lrZ3{OYFP-96{#@ysP$eo zsIK0G`$1N2Fvt25Sd59x;~@oTqJNG5q$OZ#-^#RaU12ZdBCguDQ8h}*jelsZlVV3; zMx1MHl;FW@V8vr`qykZiB!^vjkGOnDb;3iq?bVGL3@}QcLUmW1@Bkl0sE99yVK;G+ z6BfVdT}b{BB`KP3h>MP0t_Poqj$W>x=VH?ueb2+IHqJ`Itv0TSS+3tj{q=AEy2UP5 zEpV}`jk|4|Vw+p%xt15jE;O|N&+$9(e`0@eM7Y-Wz48|tH^ei2ASIXhScWcuBS3hX z)G7Lv4LD`QS`~JG`hm#Uh{Cn&sH^om!#l8SY8(`!&q>fchsQ114o5b!5J>ggW_i_-CcnzMi*jM`>Qt$m~s@LF3wD@#nQC=+M6 z*$#cX_b^dp<3#>WvF^8)?D$h@DvOkrwX(wBrwcAyOdvaXA^>(2w<@T$h7*l&NmC`V zj=wBgaz-_H&NkMp@KPQxo^)exj8f8?;Khpd`uhDjE>O(mq?u^cPUwSE@E$|B)!qGv z0M}3x#4Mw(E8FC+X`B#TZd&F^KMsspGbc_KpX4u)8ap%s`oA^3B-8wZvIJX^{^SI<=WmcgFS>zwPo z&J%WSh~kL?igFdli9YR%dkE_UB)ojb0!OVl#0u31(?7{Gbe~d=g{8S3t zDKGTwyh=NAE#=;_t`Ex~j}~;%_uD?+_Bn2v`fBw1UHc>jFV6JVsIVuM#R&a_K>gZL zbw&|&iK1HGN~aKl9XMnYt*g--Fuwn zTsFasgh;!ukOFSx+6gVgw9MNbSDH?wHeS*B3NEsW+Vv&;c~|lZ7xEg3F*FoRfM|pT)G*^* zdfKqU+i+bgQSI>5w&OZyW=5J@5;V6pMSO)Kz3fEH#isjw z$5)|WP@|^NE*cMB4HkUx6y{QhiHV@gEUne9u!CP`TqbqpVph^`P2Hs8u#RU?Ci>rv zthqRn&rMp7rpGn0TqaSGd1C8j455u9h1?rALfj{%GP-WtQ^0pqR37z>h7qGdi`#@orm8%(EFb|%mROVfH;~nTH<6B%pY^ZD~B-CJls~U)vVGq@F1?2&I-v; z(TDi}Aqub*wqiBl>8ryNMNbWB0mvf|S1*eDzfIdXv?I6Utm9BI%ph~Q+=*t1UK3QMmWeMm{AFo3BLmdz2< z#*mf)sPDX%w!y|yKfiAeaPqFS4$~5PoYhZWaL@B+)Y+}gdN`CK_ZEr3!@%8j)1@Xb zl+v>lC#v^75^S&9V0E(eY<&)bt0WqK_%f+i!}gx8`#cUZ`SIQauX}rflSh;r2sd|# z6s(v>A74fTU_Se0cfU!R710uOvyo(HJ=U1FGA)y<2;9Opm%RQH`moYOyQFq zh?1T43sw4Ng_sl|x{9sYp4=UQlw77I&w?;yW$sIh&RA1fVt}r; zBfzdp%J%TmKEB6j6Ei&Z^xCM-bRunKD_X$QDcVSjWql&p3Ta905z8Z;5KlR(Th}8c z3&e=reLhptYQC7=y2g&WpurJ;rh1iD-`v3LMPy0DP?3}@fv-@p`vH!G`{>IEt z_gWQY`FUY)eX_5g9^3!M;aB}aH9o!80)G2SzUVwke%taGyYHf`pVQ-;{~Po>dD@j5 z!SCOhVjSfsMECdWq>|ty_oZbwtD!a#XPb8ILl4i0+WzQeo5iR0sNoYw?r&5*F2}Tj z+UoS~Rjd#G*J^}gidHgvpgqktZNjm~z=sYLWIIix;-HTLy+ktlD2Zu#JnuD1zZE7z zYe#@}R<}nSZYL-rD}4gC=Z9su}rZ?fhH{M9v)HHfa=)Nc3zY|Ndu?82HGo#HMQKtl6+T3 zFfDPYK@LSh*G4|aAzzN5J;InWC#OHP8t2Wy02Iw*;Jif=L_{ehmt0x{DGpfXRu)>898w#pnR!?p6NPzH zm}xbAySQ#>OOW&kY)Qs!K+n|1DILfJ#sMZ;_2V`=5pPSms%K|-7l_uc&^4d14R7jh zfDzHx8TRI*Bcfb30TRRqe?^-BRbh^Nt}4F@3elV_$W4C<5f@R#Q`(~l&U=4Bw!!W9 zC8B<)0tRFo|2G!}t5-g!bBCMfd`-9~U2=rH^!w*inMI3BZ!>zJem((k4iR7ufuA(x z>WnG9_#e*Co8E6O&iDP_Bt>bG-{<4X;7{%VzTdCsRQ+YIR$M$RjNkX;_@XjOo&T;# z-)}1N5sd%K^Oe}^N!fs0TwMRN-AM~xVF*?bwswz(*AK(VPuUY*=~s3;fHj(Uk~sMg z`peZ--{{paIX8Hl>!OGS=UKE z_u0@+pcHCV<*WDptPZ4jGB0t_?s5BO%JFo95=KCP&c}Ti2c@c&ht6U@EV{#D^!e)9eJiFP^+Q%BP_-3yzo)K+~vRSW&aIcRJC z1mqx&(+3b;#!Eii6Te~ z6`I$zE@tOS0CL_hAN7;3rdLxNi*BOvjsw~-tiY-oS&vIdP>0b~pfTm)4&VdkT{@+P zRI+nUX<%>J(f}n2YW(S3#w2sAb$dzS(pU4QCUj>@LF5o)tu@sUJW6v0V+b;n`d@7I z%r9{3Kod|6bb9uEcZMZAb>@E<88nOA*ysk?_xhe?a=g6&~x2T;f z@bjgBDIM{jspK~Odgb9{eUz6{gRbif{5TmX5lqR`b#kF-ML8J%B%mjl$5uG;+VOJzq23{RwY(URAZ()SxFrthG#K5Kh~m)$9M{&phlUx_jr-aF6_2Y zAbx0ISQ#yP8je5Id$sL6TtWQhQ7J!e=v>}DArOYw&W2~xeq4T0Hy8k;WHF#DKJYSAr87MUldOsMxlk(}~est&>gF4fv6 zi+`+oYi?oRf$Yy6dA$eviIhE_t(s=!})Nb$Lcg7A=bhYO1QDnt!)2GX?Ja5 zEp3Wlr0Z-DpFHWy4{9+Wf;& z6-Bn~-}s5=h~BO(ZK}Yc*zRJoAD~XRW`aBs7d zb3YKR>Ozjq?|`8Z=;2crQk+q9co{SJbIQmQ6GlqU88a<<_mY2dWosldtMig857X}M z=JA~;EJlV#<7S@hSzN{q+z|wK)Gdp|f;S01qp1{0!yU!zp>Og8Q@?~ja#f;?UQ*=n zC9981g2@MCKB3Tf-% zw;M8q<4bA*c4q`il5D3bGeVDoteTEd2#4b7=NIZ>i#{LFi@nTu*z)iEklRo&nV#>Ff zdiSmWTk#c<*f{)Z>+%YRot%p50m0qELkL$W#_ethAeje>Papu8g+R`Sq{uv&ZEy zpGs~q&&H`%ka~PVl;%{!>7d}0-PQ+S!*=A9&X#WfvIph%I_@nYBnyn1ws`+7IC`ss zkDbzqlLGR7fRFZ+h7sF5#8`Y?nw5&$LoyvkK?jHkEySb>;gyGTI*UQhOodVD>(-m} zeKJlaCmvu+o^q~0Fl0K+%44#VdHR_y)FvbmNnxHb@k9lYR!6f#LdrldtiZL4`J1k8 z0igRG_wXUz{JtBvSQOujd0$tLBt$LcBFo=QE$1`5SK^#kdk9Oa=S=fqUNFOJ6=SD2 zC{MD44S~J#pX`0ox!T7rFsV`6Y?z0ow>+W{h**_j>s`k*QHky51_*ST-DO z4QkO=vIN+EGS*a{WtStFFq$>Bkl{O{jeiNIQ;ZZz7Q9EzR;VA(7ax#|7!zpv=X|~B?G@0t0n%N9xcsZ5moQveS64Soc8ATkdphrNl`UI@{0{NlWNq+k z5#4=GTn&M*XAw7TfVXMOLf1l8st1C?G}#gDpOM>A@N;rjjRttfaQO{pV+~`csuy^) zDfFN7C6THU?2-6zBvMnh*fL%`<&t_3Jwd|a^f@G8^qZguS|m_;lguhI7Y~_kYQTkB z`ZV=>+j*+v0Gigol$c&~<~WX|0`4QHQ-wAkUAp3N^3p}|5OYaO^l2nY zGt|_Wk((fpQY!r)>YDSbN(YXa>fe(%=X2(hrx(v>$?Eld^F4C2EH7F&kWIwgertzn z7KWD_mUjU+=5EwTPBDlC2x=hPKE;DAJpdM$G}hw5KrZvjA~m-E3ypIZJE#cO$e_al z=7i}yoMoZ8x+6ph2XtwIu9_c6p^$#-&IsAD=FJ35zL@5JF%c!oAEGygiJP{2$K{OE zwROnUzkpl5vG#$9qKFa@IA455(m&vS9RcY>vhcIr38mnuXp^>Ou)Hl^FIP(swhqNF z^CFHb_uDAZ*!c*@SGwPAgX8B3vCkZ#6X`;1-oJ-)E3GR}lb#Aj_3LWtYavjx3F@pe z0tD7SCw7h9_wLlCZ9bq%5YmjTj-Gzg_XnOpvZ2Gh5PkDpN$e8_f~GclMxf268wC5a z#sE%yq$H^vbcb56&1noB%NK*^NmtajUIZGzevn!0T`$r`f^z?bXf7xN5>p_@5KO9+ zfpsoaM4C=)DYh>pG)+8h(UnOd+id8Tahc$;$%;aAiC+y1(KQzgGf$*&RX+yi>wF>%1xk<3{4J~I&0Sn z30O!aduvIi&UBaUvT=H5W&dEgb{2_HT--o;A<|l=GLozQfu#z@G{!{a6fV-c@K2`5 zr9~1eTV~Ocn}%U(9Zgv*{}Qttw}|wj)s*6d6SN=+Q6iy5nto_#biGRsnCuYplW29@ zrBn8(!&7gpn?zXi481U>D2zY$RDIuQZVYy!OCeJn(ENJ^!zhU#t^9Xs_HrUwR|^do zKF2!b+Tuu{rZSzHdf!>)tdNcr39nci6Q`tzpQGYy0((S$Ot<6fX$ODI^3dR;?KoSv zUbf|QVf`u)cxJUafESHXc2QUKvci=5D4=9}lGfg6 zhf~^Y)L)K*Wkno6SMv5{7&(jvn*duo-m;c!j^*{zEs$P$VZ;WEtwyuxDH3xhrjuP{ zFg>-=^!4CEI6+F{o2aQjUv`*R#@_}j1*Qq2|D>rQ6)n8w(002`Jq~7bzYpw9T9`!so0UMr1ymKczZLR0`WIblG3+w-ACCZhYG)@ccntQ ziM)XiuZy7~Ca;B4_u)+QW#b?BX&n>LA(x|HD`3oNP_*NAAQQ~Hu^}=QH=aD%yq&L{ zlo6LZx2QfRNceJqPSjZ>%~(3xk!pv9x+nD{DNTH?Az5F&G)>qwx=N!f(4eh%UFhbv zp|rrA8(lQp5~6KVi40%55=of{rn)*lqcm;#CcyW8HIY8J^-zdN-aeEaj8#X&-2pbR zkkS`O&Xnd}^S@Qik#=wcTYMQ%z#HS$)qg@Q5+7kDFl6usN^kuZpRDlt z@Uga{p5l^nQ%YnmZ)4K<_Tn(4SXpbFk`jHnJqk|NshKG|D3A+F>%Nuw1oIbPcy8up$?-yC)LXd_*Si83!!6Fn00{VxY^?{_ zTzrmoPHkBPfx8(VviWc*rP;|T<`7QeRXR==$10@Mz*am_Y1Fn9uQ0o5w>^LK>TTOC zcg3UA2TD{W37GVJC$WXAGxyc`5#Y}sarIy+MU!sd%6uqFRR#NHvM)80sw$(+Mdj{H zR*GB?v0T}_Ur_&`7aFS4o%Zwg->)YX(vt`10+c+`UHTGGkC3-0c5)r73&UffcfcCn zMh+2)>a??2jkpRQ{vr5h$C%e{+6espn4GD}BaLm+!`}LJf2s&3H%YZUYT`rY+OZSD z+iL5uC2VOvh7S)5nl>HqLEY;qvS{^Kgc?`%W@|KIl~>2#oyfD014q#D3jS!IiY~R6 z>5BO-XX5{0z7i7@>70bj?cGMrt9Jpwgd7A5_w%9MxAg4|=$isPCAVc%@|K$?nip3B z%{hw`bnSdvA$rRr8?IHxS&b_&ivEzbl@^d^OO8g`d9D#4?8lotm5xfh2aob^tMv2M z)8%Dy%^8+e*7|;By~=FKOrwFOGD?-w4$CUo2LdT$fdthj$*>h?r39~9vjzfYa3MOK z@M>H0A~Im|eza|n{S#3%tlAo?2?!7O_m{x}WkxObuJiXHeeW!lrP`dMfGqpt$d2J( zI0j`8b`YIgdFn47@_3VmUXSN|rTJeo__mEbCcjF%1esanDrCEV93m!(8-3eJH&r%d z-xQiC`gB6!HTdPH)kzz-6Os!X<$hhXbpkY^ZgK%u;|>`W3z-v5&-c4I&8@cwB50%- zwZ#*$!u$y+H_eW#bNphn5k!xQ1GSesNH#A;KjBrj1YtD6NPb{#UMI89bh81=IA?P7 z{Ie}{;>Y3h^Za?WLSM%mhBBE}rH#g1nD;lI+(;9~vfy7mCZm~xKKI{DHqE`IhPnE- z$u3R8xE!eQzu)BA;xUR{8+zB9w5!BUpzA)pAn$_Q_;W|w;uIw7$2J%=B&(pg1M!{MTFK6A| zL*U+MQn7ppI!4U_8Oa`?ac3ko$;%Ov(h{{Y?hfV42;On5&cMa zmvKMCT8{~AB8&HaEO(47)5ErQZT`myn*T7Rp7L#{sKb7iK(>tJqGlX~9uTqDvIn z%&3kBz4h|gt^)anZXY5gvaXV%5~>toZcY*pvt8xLrOh~$xS~N|;q}tkNj_Pe<;&ya zB|;-+8(cZg`Z04w-nK!*o;mqG-eD@Isnmtj)Jj`_&Z>NO9cX;|;Y6>Cv@{+N7{{=8VV-4%mbCJ^VMGb-eO42DtCl;5~hdG5wQ)*932@W+~R7VraV-! zCTm1L{0|}P`FgN8&hPs^8Ml}h+vCsw$`c`i7x3$iF_Lq+HZ8ZU?4c2&j+(02d}(chr@HKoC?)` zALKDXP#78ui_{f*nwk}pp+ES>`d+w!qhl{J(i}f}C^WRRgMha0Up59{urX?(vS+`` z+}4bGYV3BE<=YJ!`+cbozIz%U#nf4Pi4aO(=Z$q#pL#}zIHv+ckz zTHV{Ue=nv0%|j@(;nt2_$$oh)r{HzrjCJQVNP`vsd90kpd>-p}5n1S1)Wyc(Fxq9i zWKJ++XJ!k?OP$DUNm?bQb7{C;jN|r#v2x=Dp5C%vXv=WmGq3-#I zB7^JcCzZ0!y;k_(PIdzt0FTx#W&khn$2#`MEf90tSFeM$gBtu!Mt5KoFubh6G}=xy z|G{6KbzCI-3>9v)9rps*m|Ffecpf{6kO|8ngCTH22MBFG*9|4i4x=gfIh}~-H593H z&PKBJBAeh}T2+=Fe$oY0-T--!BCS?GG8gbmwY8gqY}i{DpV83-jIk6NB&Pc7aAlyE zu5Rm#=l~b^8wz*%AT~Sw>SI-dvk7w~2O>M*Y05*4u28brR^V58$-9{5)A zSq?8T9z1idK_0X&Jo2h4Dh=J}3zoTd0(8!==7HzV6{f7*?gJ>6^8P=@&M7>yCu;Yx zZQFJ-v2EM7GjS&N1e1=Ok+cT1__ZUT^F@PRwtB#DCmemP&TErqaT}POxA47O zfqLKu)St7DPPIR_mJm!O$LW@L04~|&9k0(|zgKtku!lmuW!7|7|6y$??qU2^6-A%Z zA-}s*wa21O?(;yT+)tm#DV_fp6o_*&zR1AkAeL>oPM~D?kc-6`+9QGx1d>*%o$AG# z<|IDpxJ0uT8j5-@<{Tvw$qzPdD`-Sc?9nvn;nnq`8The= z)g40F?d5xTjd!`$C>d%+TW-HKOvQ3JjsQ$37+E*FW{552q0M0|NlcJ0v!PgHV_KAp z&bZ3XvrnxlHQYW4@4Ry&8_D=>=M#TOJQU9tqWYdYXxxXgQBga;>kfMUdBW_zC=vb++$4{)^HW zB|%4hD2iw5Q1=M21mTd%SDd%$<3x^ZAurkXy-qr>D@H5^{LOa_i5Fx8mLrd z|7*6!K}mo|5fiEDR?7=V0m7m~TGMlSjz3@Fkr$jSdt-bjv`WXGspV+3zoKKp!=t>G zQVt>xQO6b*p3cF#*A4VTbZ7~Q?SI6#%T0AcQqsq6Sf@iU{~X~G=icN#<>1{TOoF)| zh+LPn0@e>p(AcdOt_twE$^sGZ#3(gx8$Pl}11|7^jx+85%HF@v{O zl@{yRt!Wwj_V2v;E6S5(Yq5tQ=Ql1719b5x_nUYC~F zTSjjw#;t!yRXb6f*mg->=`!)P0Ygv+7uUG$$Ify;4+J04eWcqEXH87#G#j#i+k+b4ynN=x4U`}+;^LWG$01ME^s zmK21v1+7mSL$+c)5&eAF#FiOz?0J6~wh^%FcZ^Gy^gM%nzf7y5cB`mIyoWk@yMM2t99i^W#9r zx%eT`-}WTk&J~BOp-r4;#-Fmc=|B=Ze`TF@f0&2GuuD5tW_oNHdcHq*)#PmHcEhAS zXXh?sgX+y>0!1%|bBkzq0fHVrr#(4j=4H>7&C9+xB7jpPE$x~d#h1D?h^zY7FYYa! zmHii8szywWha%e$X)f&)CfC0Vb5X~O2&Y9+y_h<7TZk6jYJ_jquY{u<7pGFYIee4? zv)S`kNsU351UG+MAxRVaTG~fzNJF13b2NobWgMDu0*LiQ428?d#4QpuW=x>7Jkrmh zM~G>&U^oaWWRMypuI}pSX^zBmIc9$;}=tX|Q@(cGQwsPxOfnT!-LQRyZ8HaK-U z^_-+iT#U5nu5c35cmY4h9r26iVgE;zEZQ(6nnfz*HA>cIk@jX|xd=@pXV~&*Whxs^ z1h;6uc!);(l+CMwAIZ)a7mQ)$II8s8~WjhgS`k&0H(OdF!N~j z1Re9HTgTq#zOtN!-mWf3lD*RJ+ER$b5qbSY5PT3yi{I3yLOrARM^V0 z8Y!CxajloCN!mzAQ{4=5Ic#1Tfisx->3v2Qv3S`0$)=K44ulv)#SonU*h#J~3ZzMf zmefPn#WD^viqSOC`oB%;r%6vC&msr(-BLqb8^i@L4|?xizRviC`zB(3zPr}&K0p*- zi>DCPUK$5tQGw~kzfeaUZEb~g9$`-iJx91BL0xfG2L_87@C9zi2Meb&iefu>-}Vz~ zFuKV3I(CB?Jaj%(Bh@~MEEXyxddYm05=w!U<6hL)hDn7S9*@^@lUNo0~)4lAux9~W)R{=mk7`+oo%h)!j8gH=`o zx6qpG@dF#2HGpU%ic4+wxx2y8{x<3FP5Fx;9c|KY{Sv?WX5aFICSIJa_$?$=Av`V> zJ@l>u+YsE=q;y&%bvl;FGF6r;lU23`F3&wmCF!XJw02g?%B!F5jZh*MKOBR3>sB6M zVh1%IA8mcrn;J5vM>H}KR#4xbkOtoPm$-%J3>#cJgv}~{TlGXoJAjqaN6v$!ZEXFfY~gzyS)rmts@HWTW(uJ(F4ZEzt2)jWkx?Gs9%O)2G5pkW zw@xG0Svq=~nOw)+0!v5vs(7|3_Sw2Vx1*U_q|tRdrITDC-e9V0*iuktjms@R8_N$bm7c8S=}3dAoh>$V|I{uRCRQn| zobuF)1H1Lr?&78alC$aN(3?nAls=qV5swK(SmXv6XdnHME6)e+)Cfv-^f4rQD@yPd zULh@H9lj8&(}t#uSwCX6$i^DhE7Ayct#+;JZ;%q98XMJp4x>eDVH)&8PL1l>WKHa3 z1c5Pv!^BY%)@~W*e0^MOp01Ba1svqD=zS6JQa^bt4MrsG|0|{b zZ~g$c*E2urIH%qu;5dNWtn9N_nNx`d1LzYrwWJZGz_;hvQWJY(&{SiprVG8f!|y;~ zX&jB*8@M6fu_FrZ)_7C)(~yJZDTGG@TM93&aHC)(#l+?=e^;iYBlbH}ALd(dnur)Eo15C`*2JZisOAVU zodXX{_Uq6h)J#Ih@|bC7HU6)Oi)g6r=#xM#p?!UoNodFEpGW8th;~R}6qBffq9x2- zkOs+FxVDR$kq~_LpXdb35eF5o-#$pBF<)@)%Y zEn@Gb$$r0+!~y-S#0Z?lIkiPxrB;hV@ac}9mFzoB0YmlM8I1xvOds0Ai1|HcgRXF2 zFL44XTTF425q2jc=4l`%+2tDzkpq&%D^@MJ$kD*(*di+Uo)zdI5IocN4{|z9G8rR= z$0xPH(ZoWa8gb$y#5$pvBGzo_b6;fiH2U*V2nu#nqMNo&8#C(^P?mkr6*G<3m~_Hj ziOTWgV&Ce%hz5BR#ZCUG;&U__bG+J+yOPc99pYK%kvSC-i^=58jsu#axS_$UxyD!X zI-^cK&xcPm5=qhax^;KMUx$?G#B>Eoet5HmwC;=8?PAC6=CQB#lQ@uzFhkz1k~ygM{;$( zdGu_GY%Ud9YGq+Bw}g33MM))M9Z?o_zqi>+byHfgtI((~pcZ~)^(&%oYIT4l4vI7O zo2`i@=gHt^$0GDL$y74hu)Tjtz`|r6-v!v3Smv7xQX6yq?tENEuy4gx`mpA<@9jOz z9aUsswbA|c-F&88a^QR@6_N4Fc2Y?rQj<>>mf&vbS;4=b=n(BSUMsT4X>0h+(F)3M zJbA|tL4EtO&q}L~6ik$zI)}wa!Z;Ij)Fb4rv(5Br^ra;5-Ly82Of|-3FhG^QPm?f=ZhH6olD(f|YK6SPv zZkwvgwa`?|XhL1HK@t;|ZiL?D9R9tSGQ2a-ox|~>GP10uU4*Q0m??aKP_TJ8ZUetX{|#gdNwQ~I zFsyu0!DzzZ)*TzQ&O#!gAk`IZ!PX2IOOB9!<0>iD5esVsW03sBu2I+zo(vYv$}jVi zPxRA)5I3&{pFV1Ud()!-s*cK9H5{(Mn4|O;U5+({Dt*JATnrO#hei6ll70Zq$+Dzo z3VCsbF1Pzsb7Bp6MdkE3;#DpcZjZ@9C}z6;F;jpbC3ZNTTr89BTk<~p&txdM!X0$q zSUn5X_UW5!5Tvjaain_Z5tBki>3=OI7|ck*&%vGUkk$wW!Et#^s1zDz#zlu-`q!eP z0sLH_pq6E4zWjC4%%r$Rb>`XTzq@0lU`up5LXEKsc(7I}d??2mBr+kf#aRuc^F?UN z>f3cu)?9rGVhi#K$-^;KGvUQF*}V{iDwElM1d0xz5G_Rr9)$q?k0Jo(gWft z!UN$LNl1D81ei8M7&^*1h~hZ*=^Y4aKd_B6`*{%R0#^emyQ8ubQTR{nAQ3hN-#WpU zu(K{LdEnZn%v}p*Xc4Br4W7Nh8w{aZH~HQNY9$_Xfp8&~o|Yf)$d5`l5r&jBipEy| z7;uw_G5`&}0Eo?iya4mP@tqZTR*Ie2+YY3IG*2eBx$>W}18L)otHyuR4pYt%0#TX`dse`6k`gBk#m&lWJdW zB&Inz8`^lfu9n!1tTA^5Op__8&cR$!(=lptL(gl& zg?ptSPU)XoRd40_(B|#nhN#wtzs8o7UK`bhc2Nkw^}!INV_A!e#62!{hPWi;c>*-_ z4$3i-gfNdly^)#&e$aA?Av#^r4(#N!kdEw_C>%BltUgVC#$>M*eQc$Q`f;jgu`%th zBCW?HXN%i$T*6V@6l(%Oy^>|98{@iCJqN#c2;6vX>u$WBI~5UZ;y;ff7k27C-`0J> zi8XiK*G{ln`s^O~5rKtM4GAA(S%C%Qxx>Q(zvF!ZV4_5TU2v}rB={PZ_*h7&lz zVKRQG8>>uq%__L8>>3uK=g?f0YuH~$mD{%rlIlPU_ZZT)ss#JuP`G-Ene0tkSXSfM zY`Vx$a>#O~qS1zJvxF~CqmMAil=GxjwF5{e#$&@X-*i;y_~T>K(P<$BE?Z z8j4O@A<)JSbG39BkY>q5!>4b=F7%Kwy+;j>Kqz>m4qNd=2~@RQGoABOMaq!ZcKtZx zvzh>&-6PV6zqHpr>5pKV&{qCp2TXbJHLp^%T>;Le z-M^UW147?}v5^D9%ucgpFZ<`qQNHG`l~KN<+)+NPu8>hau4nnJ`tOVPFxBD`W2@G{+#t?V%ibNE(Oj309*kL{c9J|i0jZ7J^Ee}t@;s3# zsvhEqyq!!i2Y=73jU;F6<3>auIdN$c6@hi@=+4+z4ujCE`+Ypc)Q1wd#m9B@jfDjt zTR5udh!q~K(>V%j70{xoH?FT?dynPWavY`$2#kOj%<5TJ>QTX(bunU&h9A8hcIrwj zb{*e$PlGcajWa`m@=?$-YM?4f_6P{^A&t;r=0v-?(6a}D45l)GZ$m1?C6Z)v@zA?- zPKwKEb8^X~@%^&BN#a-Ci58$>lbv+WT;YspbFQB6BqJKB>#od5EE=)id_gdNB}^#Z z$CPuNnvG%SS?so~st$^j5PJ_?PVWrJQFBOZND3>A2L7GuA9T$rg}(e~%$&9XF4`!g zf=tlYsE3?;BH9`%*G*qJo`jN@TgOiD(OgT;j=*qV%(ohwUAxAP@nLxgWsN_oTrF7x z`1&V#w+7*)1R1~tCc5xw{rJQ59(S1*U9J{bHTsD6B>n%uKPJ26#y7UaI)8&<7O& z@qk}mM{5MPPea15mw?~Mi^rB_Ptomc+nRV%U&Sr4yhBJx zr25Bs?Dym=-(c9HJHr2fVbs@=`Z!c0g4mzpNMPkXT!G@N>=&uIJvZR~`W;Z(rj4H@ zSmfNxb)zF)tu(-rRTu@CTduN51XL5l$;&;u_jfq>^P%B9KGRkyOnPKGb4P&v62RV0wJ_q^- z9ifAHx=nZEdUGQo+RT}7%8aDl=3@1Fs)*C~T)g*8}*Z?X+m#lMS|t@W|~5(62zm%qV8MypFf?gfV-X zin)Io>*PK8Ql;@8SL672H4B~^>)IE|O+C|vr?m`bq=kWqrzoJuI2N|bk6YFNJC+9=fzt|k(dSf& z)+Zjzqw64@WFY>ULHwb4^k!f!6+%!k?TGIiWTF>Fve0~1k7$xW?Zmv*Blx;rxxC2| zD3Pp@hRqBE=>zU=F5!fmUxe{aTzo%psb%)Gadu*zBZ7%O*< z&^NL^_M=5|m#C;2^W5gj`R-~F5+WZ94jxpP1j-MZ-M9*9hssYaWg7P%7PgCqyBrSM zes|6Q5^4<9uf{9nL#_smyOVCbCs9%p zR}jwVE$UhVh{af-pii3TlqpiR_;uK!W^4WLWLsLoRSD!GtZ{Bgq9dZY1a3F5{Rkzl=P^abo zRSLoyMDjfQ?;^1b$PrKvcp>{`t%Y0khy%zMFIS~c!iTJ1M})Va$o0B~i``1ok{~zK za!?mqSWuElc2^LVo=#~EqRAoaim%A4QGyS!!#H_L*ptQ)ggLATpV!PM zdgVP2Z4_Q!i(-C6bEYh!B}{D+zfb8ofO8c>Tc)JQD45BUxbPR#_pyws7Omv*M zv&6&_SK8Qi)nd)KLq;399yqGN9hP`Fz5^7~P`)OnNyCp;yR(PAL`-8G35C5VnS45Y zM@9l_X~~Fy2XnXqP}5N-9gEKdgUQ3V$nCqO=l*x&TI&$BH8@U zKR%4%jZ!llL^SU+5@?vgiY0$-2@vN_W4AB#fDI!N!r;Ct2qZGOan{EO26eV{5)OLJ z57kxWldVzRnJj7||M&)Zsxlc4i{?nmUABw0yH1V=UNbk>Xp!Tl13!{GSu`{SP;w*a`9^{6ewPTISNx2lpN7K0PZG;y#G?H+c*taCR&FmXTfvz zfACgtyeIH^___No&rX}eBPCQe5DNp`g#8U*E5KHQy*c(HgM){@TS-zBkKQgm;TFJoYe=c z4Dh(Hi4JzNI9mN<3@08wmAR;PW0gfm`(jF12@7$|uV+_$$ghu{<5$FfB+y4scQb3^ zr;nIn`7#j$x}?IhS(No%9c$PJR--&BGImX(@-3q0VBgcdHc~r{f#=+Lr`y{kNqdvG(JBMX7Jq5zE>rICSYulU+K+TP3)(e_+ERu&KNd4AFnw)T)AtzfSZvohnyxU4~6`3m@Uc53YE+Y42hlW^%vS+8x- zpzl$enZqmdL2}u&0psk==C$sk>(ez~v!~5A&(qA>j6Z$1~%{9kI-`*jufA6}Cy{<3iGZ@&~t85t9^=8Rk zx~~-TgkP;}+M+1$o4AEKUq5`tkD-H})st=CKQ&X02te|A1bKY}Z~w*olcIJ~+O+s;0hw$ zf%u*rJ&QC#JPK2#7txM*T#c1Xl$!WDf>2HNjWW%yBbmLT#EaHPqd3Wv$^>#UeRbO4 zxoz6wi`cN18Xz9V1DJ({p&bU!YRTTj_X(dPPcfnyakMj=Zz9a$(+D zoa6u+^rwwSD=umS6fQb^IU0+g-wXpd--;Px04yWTgDVjUzgYqzof5=;@K938v>GLR z(QiQ#<|&;;ga%mj2oVZx4r)C*I{9iVC&Eu^rCl4|nxCU8w#>3HA-`&glgNE8B9Ki@ zM^-yx=D!3jE%#B`9TU z{OX@%7IRB5{o75304=7Q2Z0}j8B_p|6#mCeg#F`o#3-x5@>^x=5h;StfNjxLNs+L0Vc zPWlPfvGQ&c+D}rlwbp18Q1C{hz2dMiWEs#neD)HO84Kw^vx~Ng|ikY4|q3ZuD9cDlnv8^(e;;Ke;ve-=7{JtaVP& z@V*}xWw|W|1AJrs_aSokFiz^~e|brrPBLcX$7RRS9iSwgx*COyAsc%6EXXCFjl1gmgpQwXM{dRxDk#G*f0hP(lN`*q%FFeQ0NnZ`R!}J^o&KPsBd`d z^thBSe8Kk(*O@Qq-d*}pC<>Pu+)Q56_&My|al(Aez%mY{|BbeZ3t?7sSwuV+J#nnL zIG=XnJmv^*g>2;qy8WSP;*SSEg5&3F4NOGK}Bp{UlwrB`<3VNHRi4=5TP(N&@o{( zs+lwbC&V3sDHfS_`1P4-u*>VZ6fQo^mE5iPE#4Z#D`o7PiLH%}Jy%~N`XIt)H&gB; z6KPC*^l7h0Dvlkkli1`{j1!i``N)g<9y*~dPB%(@&5}HRH%x$~wm&5H+UU2SRE|lQ zkwsvp_NNS`2y8__u^PueMR7{LVp;s?X{m&faJV zg%eN`)^%n4mA^O~9a#}8UPNlg49nPE=p+xp^y_gs&LKVcIJ%<|Dj6VMk*ymh(`oUX z*vx%Ldg;`mehzH}t0qg7y^LB?dgHLXk}luUpy{`{`-#Ri=@});`$$a#e}QLO%+xlH zi^XzV{Nr0KCfL0GPnE9u*GrD7xN3j$BM$@ug2BYF}+nR{s;?WBn@mbR(Qz2xPJ({qD|R{{H%X?c2|No%Z1Wj;6i` zF5kT)@iNGN90&hzR_5!pL)Qm0BmMU}YyZ2B^u1&{Dx_)46|JD?{hVgOBfySpmKo8QTIa$kaIN}VF-;Q%(@NQJ)`Viaml=Z3zCFszHEJtDcZ!!!2-3!fVUxRq}rjMJb2NVM*IF<9ZD#xk)y3@XtE zl-wV-bk?GYIcm+D(&_Z`ENx}|Cc-N0e>pj_mp9|`Ed3|}7v(8AgdrQesj`!-!L)*J z*g-*uQqhdi{wBf@MeHiC<#vk$dg88r1iHSlRA9g&j>(E*;Ta`RiaY&SeXhg~{D5yW z)C0h>4c*dI8l)*+b+Ki|=AUqRZr@=kY!)Tykzrej!NSV^MZBFk&FPw(rMqwy)Jejn8 z##uW~B{OjEV1L$-@bkf(a(%pV-7b_3)-6gyRAD~{IKZBPGIU~sQ=mX1{bWK&oZm^lVU%5MJ#Odywm`P$ah{<+A zZS8c76@gXL!srFB++7lWUXg1@v)TlVqK1Z_gl}6olMREUB^v5$t3=La0#dK9Y6hOv zJDX!w6RWTthxLS-_(ng+WrjG->%X^XZKUOIsLgY2QSHUY@p?iL^!DO;2aPlp2>}w< z&TU;jB(Wy7Fb9m+RgdgQ8^d3EPa1&-eTKb(W-O1ENV5qRtJHDozT!jXCJ?FZMt!jy zZ%D&S9G)~jx?ZZPEX8fj>Yq|PCf3V8>L%C3|2<#+ZPc%sE;}yX3QD3r1M)9Ou$!Km z`C9J@gk}`|cHHu#QsX8{z+||6HJ)Y92#h7=6Ixn#P$hgzjlS5?6BudrH3f&qk+Fl? zWET$K(9Lv}5QFSvhY#f`<(jDH9qy%gO4#T%zdqbjltvR38x3ZH=6j5V96MSu!K+Vk z+4bT$#lB($EJ+|}x}uPscs?XNecTCuwieonrZ5Z|Z_uTQ(xES8#!2*~1ZxDQfEiZl8kb?F6dyD_HtG zj4|IL%O|-$f)Uf!a>HLh8MxE7*P~xZH&;g6{o)P{Oh56h5X)j{+>FdIKh0D8usB{e zpxZq5>!935r8I?pPeN*CTDQuL*zf-&gxU4=T2yw!C@4v#r<=1!`CUn9e!j~AcXFq@ z&21$KKNeS2MkSh*rQ+-+YT@cNh}X7HPA%##T!5K`=(VfV>WRH!+P!~cO*d`Bi=l%r zoCkH>`Se|@)=W-~dm-Xo&aYgxE2ZfRTV8GqMbQ0myQ=Rg0nxIw(OqGy0)CIc{K8^C zd%jU_t>}?rqi?neoZtC2N%c6?jfQS|#gOO6rdoC(cU#ULk}1;)TosHyejKorG)ZE`+4X&;8@_5ycx z*yV|3(VIJVv+sD(pH+F-l_1t|6C0yydk`9YX?rDli*4lS2v-AFoIro)X~e?@2E9dJ zCQ*+gWCP7NCC`Hq9d`cU_nQz=w%FhKU-am^Ps+C2BRu+6EE{S$sk07+8|4r1!cc$QaYg}LxO@JtxDI4izS(N**>;tS@cKRR(E{UBN)5Rs%YV3n zE9@nfyef^(jtv#6=Gp8aA!7KoU=~<|c?8aU2)!$($T#7bD58nLjRF{N&|oI%P8I{c z4nfYQ@=y8E4Y;dfs;|c{CC8q$z(=Da<~1Ss@o$%wi385b3Ihu}qgI^G$5so7b~ocv z<#VulcSQYd%xY&dVSZ(N(=4XEr7)#14@Sg3r~u?d@n->bzR8BTHsr*9XyTG>r|gTT zs*hP+e^w!9_u9*7&oE}rn%bb=7y3_Ud2QcYc=V8Sw68Gi4Ye=U*DR)A_(jX8iCOirCQ~9_P0H!%$fGDnb|` zwmOT&%qp$k+m%lO(HTOt{4|0DLOb`INV)Hq2@(*$N0xcCh@r;DRqVTXyM%YOZ!7sz z=mMAG`NHOF@agESDmh(0Af)bed&92HT{sk2|Mj<<;K``3Civtxq|p>ci06cZpy{L< zUEhor&^16&4YAZx_-CgwMGME1S|gbESH+6f#Y~4 zdt|5M%RQmSdS-4ut3879MzQ-LiIV=3td;)TjR(&!vL85VJ5Kyd^bbWR6=lXgi!&z% zx&`pA(EZ+0ZMV4H%M*`bj!Vn^KidILnC!)99-$g3{`d2I$2w3y`L(oC$mqf*cwm_t6%Ta6vqV|Rn@N!fe7Cg$M4Le zMuxEkUO%vpV@o#E*9Rgl++hA{_6K*dTWb{nepuKg@u(FRn~LQ zA$O@_Lu9{C9i=0}W=by~bbREzC2QAl?W`evM4n>ul&#lMLTw9*L)#~;Il?%J8`;+B z%p_4VFzIvI!7ZEDvP1-P18>7(bU*z*2ept?4RP&pTgHI(INwHvCNXRs#zzYG2+*ny zueW92erfWnkmJ1&4RJKx80?c%K84UelW>Uk_X}ch(3vfBR;Jmb7@M2CI`xhAoXY=J z@ceX_y;P+k*#<*cgm#`l?Ud!mF90^nQX{UBv3e?SRj|QFpD@{eCkQ+=Z^p38TrshU z2VbALUlE&ye1{)jA3|SFlw97xupWaTn;Q9beA?E;vzg$D53dONLoh671cb>LW73yk zR|d{r0!=6NfMhrVp0-@r!!b=)I_Z3C)1UoFwOmMlb0WgdQ?TDT+H@hUAZrFI0U`q8 zuiV?+iz>nDW__vrR%3R9ke=$bU(R^%hr+9N-ba>vtsPKX75D7!RP^w-SY&WoLY{Q=g9%4 zYL+wIAn>zHbwp+;dcm@lQk$8H_osrC>w?_%21~FR3e+63uP!ZjussO?(&OiH93#qy+ z4K-8U6=0^9ZQ2@2+71SzqB^Wq%%b1^Y)1<>p_Z#YihjgBqPJOeNRxT`!2*9o!?LAC zevaLV;!$Ni_QeSLdk-mV<^Fr5h(JAGG=qrZ7!V9u?dN$rl(W$_k$!UYBeh1kI}llW zYj!=E$^Js&IVMhq|4fJWs_Yf=RB%j+38HTfeViC6yI-V}1e)1P5w5FoGfEe?(2~iZ zE^3sh)1+&Z1u=3($6t`=V*4}yjzm*0{&aCWH-G>$;XJ>&lGkXFmIew8g(P`%FUQk= zqIibhP}o5hW%|H$2ju2fq1ol<-x)O*=`C5auj565Whjd<^w9@Bn;qug;mAx5S~D)D zvFzA%63f5p_Va>lkgCEdmJifX$~QFMSo?k5n(ze~W}zv79)8dQ<#@fmJL-;jEU|mc zJI0|{yeY+iH3K~phrFABZ)rN)bF#*^jwHtp&f=tNS$1!TE-V6M&-8`B>HB1^R_*&H z`h#F&{j44s)54)61+kPr%u*-=g_QqZppGKw%4?5|NE;YzZ;Bv%pxf5CaoSWATp6BP zDYi~970h@-;OHP$svw1z@qW`ox2%be=fKi5k}q6p*z&__`A=5}k{}P1br3O|GqP<)ygI zG-6e%Z#N-yzf(tWL{9mKFuSwI+jWXXNt_`gl7)toUEqUHHZwk=EA&dZJSI`9akjv1y>=KMKQmJ(*Wvw|Gk2JlEH zi9@ue%+tR4I~#+hUuwpa7QTF9UGpm$*w91ab@3{apjOj`@(fbht%hPouG-M}O0;lr z8&|6b_2bs=r{{1@3sBSNNuP!Zlly!>)^@Y0uMV@wH<#iGT(s#(4)t%Z@XmPI%OR() zZY{idhP&+)Be_HQfXgSKw`Y#6?7mh0GbN{3ac#6HTYgQ(tx<+Ay)n7jYEF^r()|3z zz?M3{a7UyamJ2VZlg|9ELf$ZCw36+Vaeca|Mc!w;s3kVdc219~3o=K>Dg6P!)HOI7 zQLEptslj)hf8+pvBg|2a)|W8&N>K0z_SCjKLV{i&&1!rm_y{4tm6c4o_&SW^nsk^) z!Cy4C91c(2V|iVlkzR^g;C8#m=jo!Nkjv+?AI242$_H)3>eyoFq4JG_UJVdXX#NRD zpTgfP4@ec5#$#VReD1HwY)bDLAKw`(?Uj(FoHv~g;$Ns1agKP7wzvSmg;@Op>!|J5 z@)iMEt>Jo|X?~0Eq`O0FM{mDv?z7TbD+FuAkIK}*V<`Q3t;?m><{LRfL!6^P6&lyU6w*xEm*sXFjdt)^ z+w_p_3QmUiiUqz~czHWv{|661#@kIHITIbwlp?>P%MK_XL7=2HYGndRn>d-UW;$I_ z{TcCWU>$7ATKN~Up8;9Q|5T(&#MLK|0rFL|BV(Q`5jf_d^zH@KJIc-GszD3GDIV?l z+gE=7t>R|;SU~UmDl&Ke4Zche-;D8eRHS>G0W>mX>2eM=^P-zw#=y(Uh_-f($7YlF54VYdLQij2O;kR2l@3#~q1a{X6DRh9cH0HzOYR$Uz~;Aveej~T z=l#^!|Hk8S+p90Iv@ftW_-iWUYiW1;v2PhVkRb5QXN2P7p-Der00=E(u={UvODFor zF1x*(qs(ETufc8s<`;XLzUO15YcnVx6vS4Qz84{i2E46iMCdnG)Bru6X64%E)oL>< zm~4Ze5@Hu5kWpa)mhpD0)Owe#S4OHe?NsiQGkSQRw`42zOHCHb0?_tZyM_-`>sbOM z=0uzujW@&%319!vW_~=slR);*jcsp%8sXYdmrURrRM5gyVy-u2r(p;cQ9uOxY#mjqm4;hOh;Kd zy{*$|#Ub_Fj60Uk*}+xLFu9ojA($-ZHQXl5W_IkHs3M5V*PAcjT+fl^sg8mP;OQ3D zn-8H|2(yg`>QvK*p+#JKEV}R5`}U-i5RWceA+19TG)_Zw$H5|$~#PW7;n*xq@B-v-rW14|h z=6UsO!*{LD7NG}+eqgN z`mRN}mLtlfHCSg?k7g$oq#Qd1?QM>Bq#n>H7;qn{0wvN(ogcDhg1;ga-^5%8Bz&Z0 za{>f-&LnLJHVyHIZELGoUfVh?&H%uU;~Mm}iD1(s`y^$3MWJijbYNF5IwicTMQJCK zH-PPNE`KM5K3rWRE`u?>Og=T3H39J>BXy%{?A)8|-uQ)mGWFG)<;ktaWAwXf`gK`b z-DEmI{i-_(M#xrJHWYTAxA%50daKxZr~KCtTw#*cHz^uZ|$4|)&!kzxt$WB>q&>zaXpbH-#SN*+wo)(=iKZDA937VE1U zB{RWu?;+XT%b-vPeoL`oClNU+FPx~~E}S&Sejcrl%;mMzo`21M`Ct0I;1@=h3u>Z+ zwu|Ujh87G*2L#@OH%Ygm_8W!1FD0mZQ4e6MpYxUb)IQ+6x8H#qLOvxqm`q1G8qVGk z>->B~5|Z&qR_Fe|&6lXEL3uhJnPuV6Kg7-$zwd|@_-*%8(0{w;rTtv^Td128$VtRm zWgX1xOY5|TZ=Hbp_Z*l@vtaTwDwP-(Rdb|8`y{3{;#1M=*n{I27Gz;|*b?k+Fa>U* zt>sErdNTV(ZH!<&BBlGM&Tm=#@cTYdJ=GW*oj)_mGWD@4o2WZzzy5*#e7g{=CxLQ9 z;YrsgDF!0wXz!^b6(yd1gc#2vzeb4K_OTDBlcPP)wn+$i2ZOfhGc7P7>@3dU1L?l5 z^Raj-?5MZE4a|)KI0I*qQv~Tks-35B!*TfW$c8n{%cx6sggrf;nlh2To)13+R08&- zd|-&%KQIev)w0)#lJ!sqiwkxm+d@I8TZZTYHE~4QY5xyn-xQu%u(cU~w5* zY}>YN+qP}nww-*jolMS|f1a7kx!ARKRqfmT)LQjoB~!dIKu5`01lreL8JGDO1a8aa ze5*xSe*a$k-XE#>C<;WNl?@t!8y0RC50F}oq?EW;I(rFbMU0V`+&`c$>WO2Oo(Q}tdHS)&|F7fP#`ehJTF{*-<_UDvt9Gy-rklHAe5NkZl{@*ZS!OPD>C?@f#+Jn_8eC857>=E+D?<9uwXXN(&t#857+Z78BjcO$i?^uw%y6 zsW2=%3h7UZvVsZ0$fF8%B~fmN3yG-4ixskl38^NEbt;AZU47@zMJ_Z7Kd5xmkcmW_ zzAv*&*5aTU70g02+*d|vxzkN+UFyMWS+{o<@^jQzGAQ9Yy3mg;;pVFv4gPd+y?Bdw9-6cP%X^DOfxnFtWL!Z9UKp9cAN*Sfg&qvD460 z6LC@4g;wSqFaEBbX*l0)5-(1k#^Onq_K=Vy9#Ryhy6@OLv{!yGx$O5z0@*Q7LE{W> zmR9^Zn54qx{rq%UaWFvzxP&s`bw-Xf2PT~an(y;~rwUqyz z)os5bmaV*|1JQIJ+&kr08zai_JaS zu(fiQwzwvqmGvu7obPmVg@ix~a8Wqj&s^59p^?@4n+K&Oy>)5!XS#QfZJ=ZapiwyP zgSL{V4(NVg3pVB|9BXUV?NByr^EKAH*bheM^}l9y_aWC@IG?01$~y%?!Co{tU;qSH z;lnszw0}bJe+E~H9%DC?!&qO@i?GJ54xr|73!6T3D+E5F=Hc@E1aj=WC^<#1tc5YM zP9NJ(T1Hfu-mdud7yWxK!@vM8dg20WdadHL>&mNr0AQ4Bm!_-2Raf+}p0(`UuWFu= z?S_14>YvAMc5u4O(>=nR9Lh51to(vl3*Jr)RyYUzsLrf|Gobh^C)7P6o!+LW{f}ky z(!*zJ?FZ4PN!hC0AE(vK!42`wOCPdZXJnfJ6J#4+wOeOEEvM7!rh5^@N+#-qsUBi5 z=UUjy^HPqBwsnUntdRJv-S1W-OdanHeRRGEoQ><_L)?$tys|#L4|`>8lfP?R7s9BX zkFG;tTLD?S_}xvKkoNM=Z`QadM%%IjfV~i%3UXX;boOUe|5ye76dGnBu)BH0ST4U5bQLz3!Punz zq!n}*P*+*N*n$!m`0)uUUS_D&z`prpWW8JZCA373{=-h4U*nB2{x)FyE^1(l_Tg#d zK~%Cw)_Ev-QN0u>VegoyrIYElkz>BpaVgZ&WKfnHjFcli=9EobwAb@xVPd3ybf^X8 zmR+NrsHAxGq`Z}7&f2y#)W7^UvZZK&S3Tv)_PI;o|)emo(#WoV(eS)aXLC2B5sXM{kB?{=%x1gQ%B?kPQTXy?wjEmc? zg#7y9FzXBbP9`P2j8LB~T=W%K^p@g#Zm@t1kM))UB0Nk~`fvHstIC1gz&Lr}k>)t| z(Nk%1(x;0BKepvM%x<^0s5VE-{0L^`*VX|7LPIG%bg9wr|Vm@*Fi`GEVCA|>*mIug2 z=R=()l&}&`;G)j3F2;7i%+n`}!|9vf_7vh>FmLcz*Yv7ON{We&5I+Tlbxe*x+#@}n z_Lc`Btss|)X)yrRT|ee*^;xW33=P$HX zY~UYrdu9OlecJ&;z!56&;gsHWfC%j-mtm79E4s zQ4IHhj$6q(7+Bq8MLF!xstT^A`a#kzSr7fil8AC7UqP1mWIjRgrfGTb3;%FK5Rg3F zxjD9=vcq$vmMZZ({8^~X9S#Y@}7#zyfV+^3=LU_!Cao5H3ps8Lu3cwNj%)IKk zpo6Jr$MZHBi$#*lMKHzm{!P4N|@MrkX zNs>VFvt-TN({?jlHh78}w(~5tS?*N-bDUgT>?&R~AAO)P(*#6{W8)|d4Ru9(USpBumkzlK!kf{N9%eXZ7{Yj;jy6)High`__LI4noSx-K07m_kU@o@TdIZg6r_B2DZ>yoL#p^puVVN!EF z@Dnq2F@O*Wrg$^3GEmXlD)T?&bt~Ki;z|;#>URgEhHeQ3!R{+Xz@2zBci(U?JE^3N ziIZJ=*!^Ta+2UkbhVwURhV!z@WVp#>`e6%eXdxBR%o-3+2?V?U3Tjt53yspB5*h<@ zlMY;LcDny`SMGD?FyIxUuOAS?ZGK~u-mB~O6OYZ&yUUMWQ&-(N+C^m9aQf2&Is~7j;&?TnXfB zR=A?3|2OxU?tJH;hat2fCer%W_~f~Bla`{?bhPjg(s}-@P8=Eeg{q~HP)V06CW|4> zOt-%P{@wgqL_Z$8;E|ou1`}s5{Nu|E^6Pa+4*qDZW@5;{Y{!{}f)I+Aho}C?|`xD}PV89FTHAhZDi%35RWH z&+`r|=)cU}k|{qQHrZdq)aX6c9_ARz39pLR3bJP=q{f)R7|`O6svRgEr>m)kPvih z1uU4wmc<*5C)^bgy7!Or7!C$tg;*ul5yHH*wKiQH0Z=nt~j<(YZCsm-f*!kn{<~w1klAv4iH{wJz}$KU>@QxcEKot*+mn6+JyJ zNiUL9?vL?f;vAe%KegF^!cT36>|HO@FEBQgN@r*C;%#(V#3|{=+U6&cQWD({HRJ7+|qOxkhswPJV-e$B3Z!{de7)rbUj{kzw zh`_X-wtNk-nP{*1@KKw)z}1QF|8KU<)yeY- ziGC(jflNioAqmoDN0;w!hH4yX(0P`;J#`m%Wr*X7kVW5VtxD&HC@tnYD}znd$+A_LX0xHqG~y1%IZ& zGd7V*LuCz8X%N(n;#IK7ks=$5mLV*hi|#Jf^Y(da@>I@t12B*gF$jr&_OOVOY$@S{ z(NkA*@vuYW*eyR*YcqQxC7*N@_Lj#S;NsHT>R( zoD-+46SI7%DVuIkYdcubP?isLL3QcB>;Dk``KQjDJ;2huyFj|ol*=g_s_$&Y|4E{m zZ=7m)Df1aS8%epKA=GDb;y$Vn$JTP~eM-+cD_|EP;aovs{aJE?$tsm`0+(w2agG3g zbA3CL)i46Z&%y}mhUb@?i*ZPf(D_$HnZy6AuWsUpvrhN{yb1qC;{JP&-5`K4p5?dciW5f z6nz9t?P}PT*#BWUvUp$5=v&ck43+M;MdAv&CE`Ok5bou@f7Unc9DSDXxdz{UC%ohC zP%^Gj`c>W)?mB}fJn6dhc%{OBa5$t{Y@`f-ntO11gpGrI0s2rZUwy=(VkO6o@7~j8Yrs``21jFyHuhJ+({BpMYItSfjDB9fMA^EZT+1a7&ZTW1q#tkT2 zEf3>0H59xYxbwvKVRGU2>FJfpO~yqj6!X;F0Z0en6@=3p-vD}N-ag-s!Q=@z^}Md% zFlOM#d-WdG{Mo}Qxmp}5xn&wO*o{qrzA)yeicQk0DU7OyrJ+0rajPK{~Iy~pp8N$P>GWzOPuIF?>b}6Iq zpz76X4nRYWX~5PU6Vt4v&cRTXduad8^!A<2W5q)k_wTS4r>I4(Yv3)8-7o+oA$k4t zI`c6sN^lkKf#qWRiPy6J;G!6TZ*h4uRIJ2C76-B0;vFRoA(JR?fNgD%bfp#-4}f0a zz47&KGc$X2$5Xcu-}oSFHGy>7F-79`)Qa-BE1+&gS(n zSpr_4wyoz{Sz~3`wO7PYS%35MC3YuZ0ZEPA&4MSaA#JY*E3Z&0_5(KGcdLKmB5vrQ z(Ztsy3?QrI=jAw9NJCZ^wN&dfV0I-@yn7>BhAWk&z~hiEPwr84dAc=Pg=`c|5)Jyo zTTw!w`P4n&OQ2Z{i;R;{=`1r_`Z4uV?fy2^s+4N;-Tp4pHdr(F^FLJ zwiEfv2G~JTC$y@piV0YqFUX_K`l0GeYW!}f9#WiLeijnaLDQ@`S1Fc3$CSj4*Vf<> zYr-ZBFj*n(;=w`#!RMpe&Nuj5wN4Oh50%Cq6q%h{cOj0oUh#}oL#3)c=QI`E5%qzS z`q7jWzuxhE-kg-t?Y6#SI6p>AAaw9Q0N=MI0Rf+%IXyo2mq%+AIxl@Syb(-2eLkP< z-^?D5l7|F87Vs`+P;j;mddBiowh;7Ta=2SG`*Z+{L%EkbR*PgMgb1AzVKV`llg@7~ zcKyPX&a=$1A1V=B1zE@>XjnJ|yMR7=&hy;Srh zeRPwmxD3UHy3pVKXqp>BM9+n?H(8XQ)aY5c`H0LEJIQ(MFQTRW2)d) zt+)M(C1#IkZ9zL2`*xrE81#>v3+Ol$uA2+u0+Hd-_~5lTG-sPr6f9+47w-}4H!KIa8FKa7Wi-SoFS;i&frdK*;0?cu2~;&s@f zom=8iZbrgRbcw#V43~Ui`rSU1J8NGz5c~ z(_2r34hrEr6SGTtH$WVm9WHpMjP=f2bJf-|nI%@dBLV8;#^Pyf*5Mz0kNsyMMd>(C z2WdSO7qwE;f-%qk|L}kyqR6a~akj~Up(u@c!aY3xA12Ul-3ttZ^v!)?cMDZhM{zJM z1hSMvaq{Fi*;x5C4X-iNOZyAcFrM*LfHY+In5+tljsZ1ZZyy~Ft2S`txr05#$rhE}vY26R+` zpv5rhul%21T~W(B1r4VV$kovy8?v}Q7NBKz8m~1TNl6u--J6NDTlCyY8bQRs@B&Pw znZQtFQDV_&nPv@i@|BC@dsN^ISFhvV&$LdA88cG69(BR)Bifeq2Th)loiY?1Z^pgj z3x7>zHf|jSm8Eqf1X)@tjJ{f|7<=YGue3mq>V(-itU6y%9b%x661| zu#w?15?DT;!80c!ws1@T3{3ieNaC%_{##B%zm&N{Z^Mo5(v6n{yq15v2W-+&CGB`8 zO4OZn*j4#${0ZsC2F<#qz0al_RZl*QCdx6^CnWT(43F!(j*B$JcEH!))V|Tf^7Zy; z|EjF%-bK!Q<|t4{xEoPUbiI@UvId2U{zctGPcqEZYZHrh#aA(5C~TUM>p-@^E^k|5 zgBuT1Z0*($;LW6TyR7e{x7|yX*trh#^7p<+(mLjP3?N)Sz3;*|JGZTa4U9n5*!+dX zo2}&gwn4FKv(RfHXV*DA1j*PpwB*99+IlQsd#nQWTdBqX*O}PUHQ_W9a|lJH+c#qL z4u<>2tASH@l?f}L<5QL*7&tS6NJ9n4g=MY|nJ!t%V6MX2RqgC8D`mzk9dN^k#jAVTayYHV`zI2huuKY)t;RUVj%T$u${BF8PM4B3KONPe2U^Cf-;{4Uqv`?rS53# zYCFe|QEY;S0GhI>LAaBZmjc;)=~h$7mjuoXca;+UtDapJIeidRB_ONYM=4&c!Wn!Gblo?Ta`Dk4aBBuKffoL@t_;J|e_t-d16{ zaV8eEeKJYS31+qZ=Ik+K(VzEo*921Q^R6*u_Wx{UdXP!tq|w;@tkn-nC$~P!kZ~u!XazMyiFq~GsQ6w*=V6u3sDU@5;A(W>$%%!ExRM`VwZ)+ zx`buSeU(5wL8jYmO4=1=6umJ>gu3aw)!V`UY(;0RQ+P?xo;8NiVu z%G9Z$^#TKK9`dbhVY$u*QeT3P3R*?Cjyrp{o(L|#2iNUXK1oWJ*Jx)J|8(HeD4bvT zbS+pqoN7qnt#_pUw+h9hL1}qgIZ)Q% zjw#^#Q&(Q6kc)>^g_$kaU@!iB_yd74uhFS#Dyd(!m_!=;$epG-DpQ-T&Dag6F~BMK z#SQ*G?t@8{n5ac@wKHzgBZD2e!bf%qsIhcy-e`s;R*5xyo`Z-VS_gCw2H*UiS`{NRS zR|~n-2^w1RyJJQh+zPeT1u1Z1}!uh=>bwx{BHzI ztPpAceJwsF*qB+wF*Jv-fBeA92nhU9v=z>Ya2_-#+f5s zyPQr6)%7l$iyk;A8UCsy)~SZqyBrm*VcuP*6-!|o&h5n+dFn5c#o7mE4&)tw{vR}D zxyj{M2Rlj0H)Yaw!z@$#hEdBOCKsr{a8jn3G%>jbIt>_70{g|rHoLHDNc44vbeZw_ zLK^Au`C(>C$%J|qN=c2HpV@zxQt?NqB}@7loHmZ<%RdHmx>tk08Hir|2FzK1Ts8Ju zy3_eX*c4LI-|vF9pQKlc_@8t>+IxQP|JHPTUKje_cFzGl*Z$u!0pHqS-&qjfRRcZe z0W}EC6%h}cu$vQU@gijPmuT<<<3wV65VagMtMXVfa4-~2)IfITp?Ye?S~r1Lz3c9+ zqUSzC{I|s)udEK-IulJtt#`1@Rdsb@EPHq3_Ug2l?=mB%c|YwlKXEe+s|!D zQ@cVsxkm+(Y4ab+Me+JB?g;8b(K6tM+TRMOnWP-hUJSHS+O{D%53@q5Pvf=a8guzP z{?+ETo_6eR=Qur#cnK#YvD2(66;BI*D8mZEX(Xl-DTzu(u zNoEMfCQZsq|JeKV-YH1)mokRyqzo^K%NS-$A_)n<$xCJhvyZek^2*JOP8+t@h0aA4 zMC^+l$!Afvox;LLp)|7x-zi5CKbxR8C@v+mYro68K!Ul%aq!hjw!us}D8*_E1jP=w zxCBDd)dERyqgT&v;-Z&zdf1wrNs+N=h+Db$p(2zwGX^xL_rC4fq!pTJU8#*OUTW7E zrWH9)D6^d^1Z4?96u2dI{-dFQS;f$d=PI*)1NSt5=xfDbP~U?VMl?Gs#>N@M#n9=k z0+J4*XIPyxXRqP*&E_r-WA3aExbJsTG6`gP58gx!0y;7sa6= z=X~&K@To9di`&BuAr)Q4NHoqp%RQE5t|SUwaW3GsPI&q;IdUO%I8JNY z3L9S4jr7g3|anVA#p(Ol4^K69$09O4en z`gpj@=XHKQWWe|ukF@CQwxp+#%8s%A>7k)qMZbUtxnUL0o$xvmvDbKZP&{`;!F8N` zAe~lZNOGVV@tDdQy}RA3kHb_mk{LP9Ru%itw` z!4|t;GrEZ;qHDNX=O(+NwW{imB0QDepei0Qsdpq|yuD(Il+k#yx3pGug)?GY1ZXno zB2_SBbEg}Jr9tsf=U4s@0mkbNTj3}6qVDZH?t&|U#_G!RiK~l}$9z4THY@X$&8X;9 z?fow;zat0MS2r&Lx;RVo*rgzh)H_W3f_Yn{p9fA*ilThQZAdqxMt0}VoK-%t9Qf>8 zD8MB;V)cs)SrF0Y#YNBtOXHhrX;KtJGt*BT#~@jM?_;ioTJLF1-zy6I{nu29FC#k& zQ*X241XQTU&zYFG99%y2c>}fw9(=0Cv&$V9PGefUmRA(S2LKd~O5l}_Km33{;`prR zG6ldSAN&pQBVK=5-}OuuFGy9#g_@-|$XL zBm3(=CM9qkq?_0!{v1(m8lr#_)TY(B@*$$4+CM1DlVsuHipLrjwW5Qj^?=k0p4Ph6 zRnko7-BaS)I?29`-np2lC?sWe=Z%%4TtCyAzWs40r(9G2q1bY`t&W*dl>O;~Nf`-| z45%5S<95B5p^X^c{dLjQNxH12YHhC7F`n)92^q51Cg%zI3=0UJ$GT-6-Q0N^aYOBk z-hVO8=onT^tC8fugdax!F-ch;+gDTp9?Nn@9zAbAHw`( zm;NFqTW$QdiD7`Ok2(GYdHJM^1-F!RnrI$fL7Ze*g@QkncleGD~JL#1$2cEu92vgy+bq zGnad4)>7R{S?MB0YKTa?ZaQ#yfJO5P8F-rx=GD8UyMqt<%GWI>wGzEbi&7o5i^ZH+ zxC9+vDfLOz3hw!utJ%!{?@zJPk!`|w76jILlZ=!~tlj2RXD=avK5^)2S-y)rFI4ch zn_(>zqR?*cPARIAkip1b@5E2(w(*5E(D?hKamD8P$mD#k#?OHQz6NjDi{iV*fA$73 zw8UUaEGFR0i?+D914AS;uu75UBDHO`pnwqPqq28E&mg(G5dDGZzj2qI=si@w=zZE3 zV$1yJeecmoqb+xxgfjGZdBOYIT4-y|oMzUq)( zwj&64sT~f~{4dt#tUXN(2Yi&1fLuEfq8i`l)QQ5rd7rksIHP}Da6f%~piv#l#r`l0 zPHeAnX^?YK2FD}gX)x7@Z8%vcV*72SVg_#Yw)&B=^6>0@qAi2Wo~C){p76_0JC^Im zT?}KNcX?Yl$YQy{rckV{7q8!n!sys#lhmWOzs?T+3ax3L7ot74;gqy=ip9dZl7>8$ z3V@8`e2Pd5(c&ycc*JORBX1(9S%9;9Q2zGrwKVmFPlbb+3`y2CxFfZZY@)Da{r&wPVF9xnY*4VlD*FN{Kfsn^AqBf3`O=W(|T zj&UVI%G)C<_}ioR+E3vgC1!5u$}QxfJY5Up^n?v@G`s5AGoIRz$X-ARuXa~n^l!@@ zNXx!Ww^U=7u%)K$1v7n9k3)SNW|p06B4+#7i-`yw*T zW@CG-17!|Y)*!V_-{F;U0Bv%6y23-Muy zHu`T00$6`xaWa;B(Xlc{Hj$0em?yw9!6jk}%Pa{pekZjM?Ufynw&xJGktB{@S?Ci{s z%ZwJEwl?=qm!FoUJ3?tYd$K*5A5mRj9)`pvB|hq+q)xIwyhwo88MCVmX!m=<6j zC`J@aB=`o#_OjpOOYJdhOm=E@JnsG-1f_gqXyj}-UK*!88VOK-tKB<$=wN)>bPEo^ zsQEM#blxIRyYP4U#`skSnuud+po?8Vgu5UGEmG~>dV_}rSy4UP;?Ar^iUUf5BUXlu zfU6RC)B=m#1tQiKy$q~BL5Zots~HKIQyca^#vWfEX_mSHdyYRq39!au^45LP8`NF} zjPRz00T*7rAONRd;q&U#xQia(t860}fOfIH+)5O$hy)A^%j0`Y`60SMyGdAxae^@m z4IVN5$VLa?B6sP5Y-)(}2LG~!W1J5O;?Y+QL+u)S=D!jd5a$<>q@gyT#X3J{!Z;$N zZ-h%izw@)w0rw;5`Bbz9;#Tzw^&rao?8!RB_JFltb{$eu^5>QKCxU!3Xhp3@wQI#^ zQWpSA!BPt8AF3$mvD{mcC2%DZ1v?|4KYI;;_?oR_;ufQ=z~yq|Vi9#OQTi$5VM9I( ztPi3s^k?Shc8cZ6X{dES2P@5fu#v1ys`t-4K|DOkjB*w5o2Z%8SqlPJg6lTO{*p|U zZ&Y=6v6D%0CR$($m`toL<7$8_oTb)E3==v3vLd`rB@kDpnhL)XRHOgWVMvcgLO|T# zrR^=gTj4FC#EG!^`(v%w@Swq0lK%UjR5q18j1pygrOuU`&CnJSR>fxjZ7`F;RhS(n zGQRNkWi!UUc0YpywOPK4oZ53~Y`Xd?l*7oDI+=Z+MePZ%=EH;U(%V$OAC$yTTPHa% zT^OtRT4}dZuAA?$Thdbyz*$Lt!IpR5DD*=)GgvsRw&|>aB_58vhv;o}$_?{L$gqs+BOarjIy z$W^~hFp7as04Uag@36EJ3qUr|12spC-0a0Z$1-q61YY48gDNZ^z~kIw-X$>&TV&sP zEP1XLsy0M#ZlCF%HGZhp}$sZ`(U}rr+DS6~)BmmY?OStB=8Y`-AIU zsg*AT=%hnVq_ON=tlTJ#0T@kU!0yuhY!5gOTx2sKny7cg!#op&WHkUC2x&ourO1=* z8N=bX4~ykwqb*%>t5A)<0U||3E)FKxiWpFz+%2)1UXePuO_#|+>KQhj+B-$H)8SAP z0=bfM)bE3W`UPd@o@Fs^NspqQaoQE8;V)R z{g;jyA|Xc$`aDruhCX3<4^@8K>KuSgt5{dU;z2R$WTWpv_~oNCE|B?jgkxu}Z+fjJ z18aHISeokr7XBa$WG|OhtyeATG@n%$_G4G5W|;ZN6JrMtO%y;)bV+qg(bKu%97eZ1 zoAJVa(1N=;#x}N#iHAoJL{2#e%W9*hQZKZmk%Y%K-e?}P2Fp@ zNlcQ~q1+9`nTdx~o}<7yI@rabB@+UvjadwgPMiHnodw1Un^v}#$hqk}!9@YaFyXXr zJ9h*%UK4s4awh7HFNlmXM}4`-b8EG_D{NuVI}3TAT+ z8i{bfA%T}!T6XLin!R_y>Y68NUSO_E_xD_>^-ROi-&6KQfGv=Hdp%JTk`}5wWeii@ zLLV+mcfjz<9S%$P2?rMe@O!7vwZK}DOvSRY(W&PEhchCnS%p^w2&YHxL>Qkb4W02c z&HMJ_TS`A9wK6)Sb#zCiv+JKzFg>I^M*=u+pUTRVl@D(5Re=yV5=((thQK;^J%^daqr^^=*t3|7 z&$k7C=z{3(yq`{Pm(88^!Gc$=TN~IMk{?-c4KevF*5!H9utEzCRFOAsGe%{awZ>-m zB8U)g$#I-HQ(F^ITC;-`-C&Z2dvVzYsBA@LbV{VSz~^Asmj!Q6AKq80vHh-#985eU zPBxbtR}3d_DA*61A4)9f#52qsO<$I%0l%y z+MlY!%s0u4h-A}4_$rJ`CZI1eg}9iU2nr$84y{bmGaep)lCXNIDxg+z$&Hx z@XyDgIRut<>Mc5?ajK64!=k&1uu%%g1s2r2f(y=G?_t>(qXt!tQb%2P3OHRr9I6Sdrd4`=$g zX591~jn??~m&pB};|iBBin4n!#rGX6R-o+0HGv9J+`6M;DLT zNaY;A^o^UK4fiqkxme{KtB0Y!5;JC|2UAmtb6j5J_Ibd4tEptgOgBsPrT>0q*&R{A zDavX{oJUj4a6VigSJKK*lADJomz6#^a-1jQZoU4($$CFuSy#22g$X~ruDL~dAJ+Hm z1Apw@sL}`9BBOTiiKCbJy#G!^eliRJOa@CE8nBsnu{T3?yP?odk_s-r-2#=QfswuP zRjObfA$m^XSUwFJGJ$NVDpz~lD?HoSppmnleLnBl@khtSAha7N>Zbq!Bw1IYi)S04-BRV zJI_STHHjq&g((7-DqRJbZz~mo6v~|S(!%pzX10DBiwr6&y5J1eL7;tfE9%mCJ7M%; zzGAh+DH~Sf@-@zFDe#utFoa`78>4p7WL@=VF*8<{GAegZgRk>*=k7gB1<4{t*yIy2 zABI(zqJQ-A#6YZ@pX%1V+7$>{nfvB0HqU>;0{)r#d=ag%x-ZMg8}tC!gSSjS z9Y`>&&SBMSi5+ITJh)OFwKU3ma}{vWXdbGJVI1MvX_}|e4Ran$t5h!ve-@N z{H4;8>1?DJ+Db`E|K@kNEI28v;U*YHNyF?E|MMeENbKN_QGu+Yi)Gs;g=b9QDsx?UG9q<(;k>gZu z_KyCcL^Y{DF~I4u@ zob>A>$}Sf-TJwUnT^1h7!kOJoCb%vqH(B&TfgZ;tdXL%ls&q`iMx*3pvDY3eH=go> z!9EK&xX4dn=7wieXoUR^O_d6`P0a?1I7{1Sz-7q^IChxz1dn7z1~q%ZUB!Qz1AeCP z%tpLFbx^J+u8RuJdhas8;)eNKQ1Zi7!5BHMV~pGnidDVHmrelW1FHrH)WCvkyDjHZ zYk*e6n#9KVNk2G>5eMiV>)(dLuk5b4z~MpJoQ~B6<6)YdGPl0gkzVE72C?;`0%nXb zqsdWc-a-x_;f4_JCJ^m3-);l+QMwEGN!Z4=eC%0rFJpa!^MT;_fWk$;G3{w?Frz=1 zF-G+$_GgM;m}C4O8gvqK`hHNBEt7>LFGa;1a)AdoPnI0GSWoc$7fat=W}Ge)3ygHa zQtrA??toc$wq6FeK&D?Ps zvqps1!#X{&i7sF@cE|;n-KZ3^N`x2P@J#>A0E?fD{d0T@&AtCS>yVAgNp78ohs}6G zIcpCt_v)RPDLwkOJ!|(A7RhY71ztbE!IB9%XJbK zgeG;bBiI_hrTOu10)$zfy^riNa1jKdvGEiFMUavz*>U&8{JMkLszU1ND?gV+Y1o5x;ZWUDT5tUj zJ)yivGwe)s1PZ><%y1TFCeBIhJ2=P-` zIz3s~ho5zq%gw+Xgk5*8u+B_srUUhLSDQiPEDle0DXKSQy{~I&Hkn1@ax}iW zeKc3HHzJ^g+@%5>k}}vwr_=rW<|~g%`hRr4)vW(84l9wH@jI;7%&%H%~@Jo z*C?gO!eR?IF0CH1Mp#ZW4eqpp{n?!^ITpjo1gLu>H3$V2CrP1}-rv*ig zfP%PaSW<2{w5Qr*kevGnV~BuIrDL^`i^Y91LEbrO;jw=^wk+oO?>$}rH~DbXPJRq! zhu8MsDek}?J6xj#kJ8tptmW!CHeEGjpF*k#>-BbWy<#-9c$=$);%%>WPS1VhakeC-xD)b0c!3CXcUd%v=^q5j)P0NLByIB+2xu%P|Qr836?@eX&eb5cpAVQ?)WQfcw;79AVNX7r<;Bs822rhW zRIE3epRJB@Z$x5{s$4j#QZnIy(h3_=?eR@N?X%$m*Tum@!FcmXsK&XgeOdzCr#K;#hKpxm=Jm**Fa&mV6NQrM^{x8*LF5 zgsg^DZERhF?1m+;nsmp@*<?B&9`Ap$d>OAhzA&X`i#li&jzM=|K)X z$e}AUjgv#&0v{yE&rpI42yiV;*wVLH4q@+BDNYVCH zSwN`Nh5L({R^@vmq4RBmkwAXB>#uQzZlV}E#Zq*zK|Go7eFa?U_`M}*$K9(M<&v$f zRb$vrH4f*u?seo@tN`O<&cH#R#row;DLNazX!bj|Qu8vWVq7qqLhS|n*Qj;lA_Y8tFs2{sG;UuTfI|}BS&hFY;6x~h7LJgRKM+P!4dwQZ)RU; z)rQT&%R^5{H)FY3E+Q|6IXd^1oyahi?xrV^Z4X$2*u0-{TsR2|qEP`a(a92<1Wx62 zF$U9F=XVjKZHvju>7q-vLz$~FhIQ9T1ATq1%7J0BMstv@i_QbnNN+~Ajqxf4SYgmG z0d|?y*0J$gn0s4rbx((o3J|MNb_NpfB>HtdZhM#e`*uAld_}9Vl81%ZlG7l7B>cFt z4#h+3LahJV#x}QVIe^hF^F>4{lgHY{criSd$<)0ScKLaIGp0`MY1oMf5?<{Uxztr0 zR7a#!M5AMug#fN{myuuvG*`H(HSr^&gQ{vJ+zF^@P(l=UDRQQ3>2;$R>|ygz1;edF zFE*bCpSfYER^*ZdAp`=-@VOtG7A7jJiy(8y>);k*@bq!a75cPfRYilWJP_h_O`J+>Htx z_JepQ+lsZ;pLguRUNYtb1Zi4G{M|GRpv3Fy2Ka$oU`N(#A;_t_B|Q0G^IQX z%THy?9$OPEZKSBW zm{T=GsDt-c*|-zX@Q-cEz++^xbiUB>M@?tzi_o6*QYuD{kHKfhb2?+k#{i?xXo|_$ z_LogNr0LAv{UB{okp(TZJF4(G9NvaBnvR9Sdjm*NJ+o<)-EOMSeYzJ#AVMEZTIRQ` z0QioipbH}n!$+31`;1V)IBgcBz7yHfl36ao3e>hVvLati{{458FM)0e_P~nCzx`W7 zI~CR|1d(O(mS%pi*)o8pv14^0+xU@<__Sb`P+keqyKEbXw$;$@<&-RCNglAED`8?P z0RY|u^x^lA?2%qmaTEx^i-Qv8~l7mlIOoN%O>W9!`Rw5XJ^M~3g zc)+r*D(BVLv|waGm4aJ3=Dv>hYHPN@Nlup}p%8JRRC@#sdabc^tP^Hrfgu4{&I?@- zHaOCDW^YK_z%KtIE-e#GF|WjiE-LQu9bCx|@)cB| z!rIFiF{5~>M+lb=kMQLzWd8daxtf3<`Q3ElqB`jK_E|>Xon2}yxbP$ zZE2ZJr##_IBtANZ3C#-@jPJ~GVojktuu;tA13+dNazNYbh(Pd5&T?5GJfY`dFiu%0 zFxfz8Pa^XiYYS^h?igF3RSj0MrF9P&k(Zg~iwrjK#aCZ~vr-wMsUVpYGo$xAsi+`D znK9z%SXXS4jY%3A0z$rkhX0$qXu&=UB1OLZ#RYhrE~{XV(}&>SY4GoP@b5hMw+er+ zPXBb;923%WuVzeNfSh${XB=_I|0*AdoN6WzKE+m}2gS8IwKeJOGhLG#o-Eoj-_$|UL}wNhdD34n_JFXDKpj>0czIsnicn~xXBcr$Zgj2!k#RvUfq0qZA49) zGtG*XxkIpa5s1(Zj*OjOUAC_W*d57*vs|ItYd-y$&^J6|A4lY4!U}EwZ7BZ7JGO)$ z!vk5kSo~}6XkJJ%aKHq!|z$P^xDA)^0loD=4(aPt3k$8nj;}VT7?;VDU_yA za1cgsh}&{ugU6}EbVJ$J1=UA&l}n}J{!nfx#zP_dJ>{?<-@W?orO}aXoOP+#w9G6e znGm|*KcInP1m${0q~z(JBHA4Ylw&mDd(IxtY~7}@S&*?cu$@67&)IKcQlhFky(;8_ zd^DZjmOM*Y@zNg2k4TrQ82Isw2|#2K#~K_$M(%c_o^d@dZy{$Wh`msJrp|)4&(Jb- z_Vw@o>pymS*_^R%AO*O#HqeJTvvu59K;6*P6S0zatO7lb!%*yJ$<-5GC#nbEcmF7+ zEGbxv4C|iy^1}x`T;%!i30L|bNA0-=M3YVkdE1|kN5F;VeXX{oJJwsA^}u-!oaexK ze&w9Ul0#%Wk#Oz*@@yxfc~l7_^|ULh=g!e}aSk`u#i-P6PU!pUvb7InV-bhWj3!p! z%mCyk&=t*qZ0rJFuv9`)BD^>@Iw=zxqW-&r4G(H%-#iX#W!rOL?yC!OVDA4a%>AHN z9@NT%TDkh}f%6;$Z%wvf$k0iouCcf-Lba^VV_LA+9X%Kw} zd4ZUJHSxjxA4l~+Xgd3dA$j3%)tr88Y1)$Z)`hp^#0W>VCT-FXth>@Y%ZOkJQ;HT# zOhZ#;(S;=yIDEcy$v<~OJ}8?#WZQ>meLRvrgWLtzvb5v8Oo>1Y;rg&)cRms!i?#)Y z*hNeD+LF6yY+`|omx z$L>CFX}uKO=BnmuMAV4nBQhV6bVMFTq(w#S*)|kuTv6<;!{Ng%T-%y@?y;7nTx61_ zwe&A79ZrzT`D^7V4T_B8D)LP8h5gBuT%6?TS!-Jj%kYMSrKdqE2N^Re;DKC?uIt-9bdinjFLPywvnIV;*u5Qb8?Ly~SF_S$3E@ zdvWS(JNZ6b_p0u_0z`2pbg=|@Cz%>nHmSYc8EzCCY!nlSGc>dYq)A|^6-0!CkgH8= zA=632=_7L5+lMo+ zWp#;!=CTxEXELf^sY6a{HN37{HBdjQ37cm*6?DdYwK+Ae$oJVcjyESE9FFVfcwFB* z`EKRFa?yKW9pUaev6s`T2d;?Bxz@J2T);`dBjzBIjHqod5dhjzkn0_gVzvw%X&oJd zLE?cch80NKN;M)NV>L#L*vvj9MzhCZd}cSY=LI!y!POWF0Y)lNQj+o+YmF$_3Ye7i z3lfbms~gRz2W6FXAg%6tP6bW4nC;d=ZW~=SLbP~|+BAiawXq@YOTXQETjPc8wBLAv z>eLtP(}EQoE_~)xpf7;V0GK9L_Ok%;WW;PBK~$2{8tO; zqDqyyJ*2C_V|Gj0?SBm+Weve&K@8ezELp2YzeR)@mV|k0)W)qa*?$Pyafs||Xz~Fx zd+q%o^fIRTN|N`r1{MnsREylS_5milv^0H;_jc^Oetj0xc!cp_41vB*2`K628ki-q zZ-Lyw+e5M=47o8@bLjet>Up${zsF<_IYu~kS3$sr)=~#8UsWCof~jC2h5+>EyqF7V znW$}}r-;A(UEfPc5e1ngU`0;`Kl?`&VoaCHUN z2Efz)K$nCTvJ^dK%OHFl2prQR@^)pw@1$Vf1<~0~U1+~$nwmTIV)Ek4i2mQRq%0J_ zXYUzJzq5tpEpYVZ^hzo$CoiEw#%p0nWx76CpP31xhMGs7E_9CJ1W#pXzcNjnZhfu> zA^pP&uDF?;>f~s`)*~wH0`KaCzr{-n?IIXIpW18$_|-$fH4AKcCsUfK)y?T#XpjT_ zzQE`GtI(g$vU1>dwI1sy+09aUJ)v4Fd;cpMIHw}*D;?#>o-Mn*!PaareuZ=QjZeM#x-1yDL04H(tfn0`Cmo&?k zgnde~($K>#&q04#AFm{}zOk>uKV8)cWoB#Y4k_q%B~x`Aerp&=!0J~UYE^DyF2#*y zBtQjke4k+G(wZi7TR0p40#!+u#GEv3EBYHrxW(R#W7 zsRNLt3l6RhbWq~4d+F}jvSZyY(b{&0y@Ky&R2?BFRJ{Jgl~(`L1&!vnTv(|Cy7XnG zOs{UmMG8N5b>F+5ZXnw(s?XgGYPV7|v}0vc(xgyxzHp#1t@SVQ!K^elI6PgrL@@2w zKb16_)EEQ)s{NYa$7TmAxkyVRT&4*vFc8JokyFwJ^8(39I@CW$Z>qzfP|IG>Li41| zXhF=aoXMiq&DswE;7{Z=OQcBSEQB3zaDP81DC)+?vMa#lkm`$%9_+*8^6II3ttAgwwY>CehP%~7MIU^BQ;cvm*O zPZ1@Yn*=B=7c!Hx<@LfS%okE9UBG=GATcdudaQg)L_t%S`HO9BDna#08kKY8TYAf~ zYX}*UMI0e_JOdBmE3t*^;wYh3p z`4ccH!I$AS9kr8O*f3s@L>2{83n{R=m^W82x zKMgkv5{9EoaWwK#8nuh=89g}kj^J-c zn~E(FCotlb;aH5-7JSo@Y#Dnt-qu=k(vh~pZ6z?MAei1NYYM|S!IaRh)_|GI#xf~Dx3a%(3px;C5*u8*R*H;+zbam&*bi;=?s%*DKwdcL}m=KR;R zrS_w|@}L#i6!1h@F6jY1=H$fv z#2^87&&>9w z+|2s6?DW=xE9Rp)x!(&#HCI#c#s2pz`A%x*aQW!N0)n*Th0*dM@iOI8>$vSWmimF& z*x||~P%}cfBX5D}z3T6{(=3y>G;>L$?K8Srd)JTL{@5`9V7q(<5gcf9Xg5k13;W-%?0>%| zzjS;-uE91Up+2-oZjI^|1n@ zKB`qVJ=FR1tW&_81GpkzI`2*oI6~k-`TDDkJt(=q?_0A1-%{YBrwvqXl>qhZY7g}S zV&Vy!j(*k69b5q1Hj*<$Wfh)9D)DKD1ifPj{YnbCw1XQGB_;|imnSn#He;$K(#!-ZV zJb)PLekQpD^dHlVB`fF@kXCW%J!F^D>IMp)ja>AWb;gie8!Se_X%(JwyX3&$`W*a0 z?1;UNs=iGfLH}?KsxeB9ZHkS|Nf#E!1ug%IGtJ*Nm%n!Y$2~6^K48)20;3RV(UB@MCV!Pe7B5kfG(%7o<7ge8B~VFLrU(tV!qz(I=$KrBX8*FUb+7+J zqkbxvq zb}1mvRth{ZUG{}(MmLS{z~g=c5f!^T>|uMIN8NPpV!>9AZ)*>`I@Wk{40)8Lwxbvk znn^I$8KP-^#Zvp?R2D%9d8l)=c4Z_X8Uz51YUeceBDe*LIQsiI<~Wqk{+zoL^Ud}_ zu{Tvy&hld6W?;L}iaCMVSsl;Fe}q~Cx@E|`yY{6Kg?Ne;@Uu|(GlC^gS>b_HEV%>r z?q=B`3RW!;Mw2v2g@ciaBQy}t)n`0jH{V-G{RLiCk*eLorJW|ZxO2_Qb?>@?a+T&@7V}y zuGgP!e)0yrHrp{az0zyOnr$!kx;|8AUJ(36?G46=XG{ciSUT30)1f7^^-#txK2X!c z$cb|m5Y4j8)?o2!t&!xltn}J0gVLDP?OKfuhK;%GM2mppaFV-1EN)9pX8hhT$u$F6 zlOYU^iF1}MNWs!F>8gZf9@U`=5ON-B;0P}5$mUUPkB<;nQN*!yK82g@l0ex#E0$#8 zaw38LQ>FL}Gry)Tstgn{n4i^UT01zhO3o2jPGl5Ke_C_;*Jm%f%;}qG+gI>EZwK$S zhhI*c?rxR+9NwZBQ+b#D7G?n!?XOP*OZwMmU!A=;F>CSFsaci47XDRW3-@2zk#^vE z1=er~qe6%YS)?`;+-@$+0ISIvVIX%*8u8lvZEjZ^!5RiOe6GnCxh&WhK{Jm-%)0`H z#?CI+__si=YyA5flW2Hr(>V8TxXFI5a}zdDmFQC*d5eo6io994-y?=q1b$U_s47;4 zzA+2fcr0owbb%$LonwW4TF63s95SXRFUp;eXKF)sTiw)Mse74B~ zxSFLAvwv-0#ef^@A;*INIfmChLt;_Ld!Dw^Y%W|`xIY;2`HwIEU{;|q2^D-<-!d`N z^OxksCKH@9nzFWBks+afy|Qm@vA*%4O%Szb^+LEwRJlPJxkM(sBk%~sfZ6-23v+Lv z$gD#t@4_@|jc9Wf{3CYza!S$~LW!*>@5_Wzf^8WhDOl28!oLrSMj0uIi`kokChe$G z9a!oyDC!aT=_@t$T{2-IZ>fb+rQll@c<8;E9me0zl`W<%p=6p#icq)YOQPq6ENAmJ zC(|>7ao+&IT}RhMdRs4Z#4Mv>E47WR*@EiUa7O&@^Qvs5GZhVvee>-#v66|Fa0$Ue zYlty;j*%%hPrtuXQ45^<+9)neJ5z+Op{1V7!rToYZ7>|HC~gLXMaet##iVMg!wC_W ze1o#uTe=vKs&wJLaiPZm!AUiB>ubY4J6k0Eb<*Q!K{iSua`4(d6n3X6EHTA|;=r;S z2OpK}G88=9J&ysR?Vd*f)2_N23aafFWXIsz?s*E3?fRUaA=tKi24&mzfp5E~7lgZ_ zZivGjv62B&cP}1yeR6$JgJg#}NRi1!n`Cd`gVhBFFcOP*-pxgx51)uA!w_*;K&Xu` z$Kaj==Q(hm1LyfwbDp*G4-rfGFVA)&nonqZF$uD|F#r^3BStRH;l{eaga8M{LmGC> zXeb+tICN$-vHB(&o&sGFf*%kr054c7p(qBtI5s*d25Ge0q{u<7jF`tkt!#S^%zbr1 z4$S>ug}EQp%7a>YP%Bs8J#d}_=Q(hmhUcJGK7X~66?#L(GO3g^EJL`aVRq|Qw)n;_ zRZAD-aDB%YKY%YXa$dH`ghPV}6;5DBJ^EzK$+=_8?G_r*m!8Es-?8O&k3iES%rwob zUp4L>dz`4#E9UG;#MwZ+iKMpuVoj__GfbS>M3i|fG3Jp(n9anOeTXhkCAK_kktOm1 z#fd5#7C-JM3o#-}Y+biL(WF}}i4sXR5l1j0=v`0M^Fp+==^+-oEmlSfmPwj|)UsAn)>Y6|tl&(6<>0nCs{G{U`qiHs z0Dig|sVwz()9;Fu6*oL*KXQ@E7U;1!c5+b#+t1|#kr|>pmwGIxV`=(1vp?L*Ql!cT zUOIRoL%3R6E|^-_QZi5>`})a^$Qp2u_!6`5yDX-IMpJD`d{-g0YoV!uC{ z<_j-a_-6n3-?%T6xAc=)U@~r8@s8qak6%l>wRp6yfcgU8(;xqv+N>wWBNLK4wAfXIV^p z(?Z(2DWbg{Tt-LX?AD^$8)9JKvTUKUMr*6KCs+DVmU>Y(3Gtzo0gT+xz{GdwsZzT_ z+t8}({Rx!Sdbu*QeW|--U=_n!-%!D@*~o6Dm?TUoWP(&I114zi^+GmY(B5j959)PB zmFCHxGnw3(xo?fa)zoHf4Foy=@!BwqJG1Oiuo4$wrNTLF?c%PyWf{}Q3N%5I(`1ei zaTbc3eOgHC3+PxJd;<$|RJ4h|?XGUq|vez*x8T}62}0k*Nr&O-(sMBku0 zi@`6%wx{rNN+ec#8QJ19h*rv#Z@`>8+k+v%u_x_~EZk>jDZ4*YDSa`5;VJ9NtTD~f4tp|$Gv@2(CW_cq75-Ldh@SG!t7*B)3W1^=mJ zjX^ro4WX31wIsI7*^7_x$wxQ1B^fW8lNwoT+7{-2qIz{%5Fb4W_ zSui8DLfmsUv`D4K~n>4wb{9?p#FLT1XqefOHlGOErA-DdKZ*t5QiS$zNga?m8ce}C!bVE!qmYr}^jcFti(!{3)6`({PD zj1v(8T&WJo7DrS z-kdQgbmR_@(OH2|kyi}Or|C;qjJEd>!ijBc>@5{^W|{&5OLF`X&Ut@}CunmP{xcgm zfbpS<>C;KgZmM7zwkAk>lb6<95Q|$#?%*mT;FQQjB*X(`wW+*i59xIaZ zDbu`l5;1o=mx62AhD+Npo4)$-wmq}=@7+$3x4~~INo1BGP^&rfuoUjIp^Bk}wpWw8 z--BvsmB2IHp0oF_&tJWLJ;~GG%;m#a%W)6@zm27GY@hruh z$2sal7+iO<;INITTC8uyb?x_P{lvV(JGLa|tbhUYd&P?SQDZuRRX8gZD<;_dGg;j} z<_cqd&sqSbH(lHJY}&N6_SfD!?!>}6P>~Zt8rK>V*PeWFV?iWh02bom6^4Q7z5Kvsyrbeix5X562i3 zaRoBDrps_wb>0snq`(yf8LbVIdb~XKhxjpAHk)LX5PR zXi&&Z&*2H0tnTR?|GXSP#PDq8mo+%h0FdBQWFYX(n;l* z+GWJIo-k=haOJbPw)Ni4`OSEuk=r59!A*A=3eb<&EPA{-DI1ZD-!U>(e<@_S82Pkk zo7P}aPq3rvOsTGt?z;{e?RDA=xiHY!ih5$po8Q%nDY~#H2)ILGr!FWZJs3(AfU!$A zSrqc0cnCa(i&Kx4)#O|?`pUoB*~cBjdh>=MGfbI;R?RG+to#agM#cd`=8i2rszYb{ z4I{3yXw3Omcw$Pi)-vW$_OuRk83syWF1(hd16gJ^rO*NK+grF=Pd_J+4X9i~Wsl02Q6v7MX@f3n&qo({y>?y{~h|LqyuDH?3v!(tl-76qrATR!{UFg|ND zWV4=i$LEpRD?U%bWbtVR`LUNa6mJ|A?FP`Op8fJpeD=mc@rmW5_{4EjeEK2`@ELwV zTr2`EfF+Db`=)+j!Nv(9xFZ5l_rj7^yc+aY$*k1a_>j*-$_xWz8H4=dK61R;P(aCn zFCy8EYbBbL?F#*Q40G+~m2pb*UISnHtYLAMup!5GxCQKStY42}`P#(lwR?-#(^$Kn zv85}DI>eyJ)-Hnkz-J(CTx|t}{CLFs80SC)@fmKdOB9c@6)8?mE_`CVi?_WLniW%; zu+z|n#1?awvw~*rZVvms%S^W@Mn?0uH;u4QniXa*F0L<0TJU>TD01>=){%-}(KK7L z>!J)$(9_OrPtcm<_97sp)vRz4a0OIoM(ZXr7gB)m1e9$l%L0A&9*Z}TMPWoW=#(l_ zu$j?63s6)o;CpF)vGq0Ei|g0>csmk+I@_^A^P-UVJasa2NdCDZ3#tMn1FDovI9U6t z-RbND)o+lxJ;q)wOFHyCOF|cmtTT2jYPp3>*;RR)aW%i*wYRXTZ46mnidjTtklR5}DXU(k$1Q?=}GePX9ZV97tAr;9b;ywr>g>aqxf9wg5Q?9o6uG8Syz#S zF>a&=)tc2`&*-n-4^Dmbh8 z^xcIiYKtQ6y{w!OE0y9FGDUH6M=nxmt4E7%nDcR!UV9C--!MWT2t!!N9A{o79nHt%=2#`}3G(l~$kLTXVxorsASG^ z3jcg#7Ow>ywdvuC7|BgDM4DqoZWN$VEwcOQz7Mp`Jp-KA#GRThRO9-yUeD3H+suin z3J69(5miqWiUD^US9TB6jdOAl5TvHjyBJ;Ylx$IJo9r>zVACLjk+;p)g@}kQ4?zZp zAcO8_&p`$O+q<%%1naQ`5o;EDBV5SAEpsxtmH1sWfej^yVWvervg>#z4|4}HSX_i5`4o{Z?*^)ggktCA{`%M=rm^C-(<`MIL?8 z=dr*iD0%qdx4(?5a)l=!Ch(6hPXBcJi~HfXr>7?;fBWX`Uv92m|DB)y@;6cDclgsU zCx2tF{~p=M>FJ-E3n8vPHEE0sJ(k57m;NPS{6;k57HE_EQt_Y)W!_29dpPPt#9ud{ z=OX>a#;1J-(PzCQ07us=zIC>|Yp0%>q*=a@1ud2&wHB|Xv%WT>g6*v?0SqtYZGhCmmY zvSa1n#crdGy67#n!J;R&ky9gBz6Ggbui)7|TT7yWjrh;(TO%!Xx}f@4ay%oBX1{}y z$K51c1e2Daj9SpXDam9)Gk?5lLK7UU*X|z2I0wz~GCiV{2`1D`VblyypsW!nwQvTa z&eRPQZTQ6VGAE@nBb1{4eKo)bSCI(-;kba0YKnX}ZJsl7?N_U^3Zb{(YNDzH)y0m$ zisNz$K^U zM?H{|x&S|aKdlS6wTl~X@UcuZStb^Xj@JtQV0Wo4gU>Y?oBwl$A?MY*WmAk{!f!w= z?@*%9#Fa4(d2P&>Uc!K?T;6w=m;7E?uPg7MsPwd!F|JTW7L28sOqW%|ElSZb*xOSV zZ3;J77(Jyb!v%(@f|+#((7r=ZUYgQpkI*xU`NU=&x&vlwA4J z8R)gT$SBXriDHbrcIO@ca*av`nl-zlr^3)e=_@}A3%KgcvVn&lZgJ;8McO>n&bp80 zL^e-Q77*0uYD6)S$|}~QwY~GJOBLE9@D|$YBk+ezidhIx>hExCn}+)csKI&9(~@S$ zce>|vaeaxs@!4FD9~hW+Q4*+#lp7;o3wP!USX5{EO>17p89vY@U>e)d+FGQFDaMjnR*mZ$|+A#igG!4oU z_L+m^gJ6X_9aleGj0m5wiTlYnZJADR3ZV@>Z1>|I%;d~p|2Y0b zP#f!y5Utn~{P}U^i!gh@S84xvCDZE#OD48-=Ufo>i7UYG!li|s zp*Ewz2|T7pv)LZYtt(WEfYm?C$QMHDFZ}6t5sV<&NJ8%=cjZwH2@v3C=z%h@TXwr7 zIaiwAu?e|00#6t-fg33Zlx1QmDTU_kIWokm6~2F1hCmns@PaZiEPYY1u@k%StsZlM zEdb|Mpjx|;Kg)B9I{hMrW^F8U#I0LP6)k`UZ>g`sYC~~gMS~C>D;YSALoRp-z;X4%MQxW&D_8HpfhZi0TC{yBpmoJ`=|{Xq^c?lNG7N(^U;Zwt4r5J;cGHM8wqsU6i z6C2Kq<_4f5!AOGUE5bxhL~qUwK0wg0y;F=RZ*d>o#-Ra{ira@~ih#&EQG<)rP;p>q zV8Bpil0fS8G^8(a!gY;s+vUlrPUcJzB>|~fX_{p$^^POlI0;ghbt*;m6$or`%MloC ztle{wQEQo+mLkE(O|F+=iRzlL42B2=hPOBoZ;B3tz^LF432?b3@-?GlRgM7*Pik0| z$0~?CLjj(_2&JS@pb^Jmcmfxjh-~3}PJ#tm6Xj052&b5@!K&iu8L`n9wA3=EnkQhd z zp-IZ89v8&Hb2B=fZ`8xm{wGN6qk zbN?P#mYM%{QOKygm&UpAnZPuBuABwX=GB~}g8~1bxBfD zm3`mMccIi%r)O*HGko5Jyp!6x>p>W$t@bqvMDKwvCxU5<8|6&I?_IxtDU6U~wTj?Q zxr^)|!-8J4M9x%255Vt_mGxJfzeC0?sg*96kZT)qw9@%HR!+(^<$Z8+;p3RdO$+1N zE0Z9+1sp`NU33#9%*(2%qbq|6VCIiHh`r*)JwXfq7LCJ>6pk%XzQ1V=}1M1HX zaxcji4EtM59Kp|TEtzq7E>&($BJ)M5SwM~~QHW<#1Mf}^7w{~ng3j33cR#LDT+Rk@ zsc4%v@%iV6i*&e1{m*-QT2`-;<>OsfX{ah^`MK?nD3D42P07dnID-??b;rRlKd%&d~+Y~WpfA%9zAoh;1Pv8~6LH%m!$e`CT z=RFRDp>|N+hZZZ2-m;ARRI*}8jL_%O1>yuWD9o#Pia8L@pR*5(5jb*Qe=>qEz$$H| zovt=ghs%Sdh(23>8AGuaLjc3yL_sdQ=ltEP_^Ul2?u=|ftOz-`hoou7xPtyKKr4Kt zpEod*JGLCzP+D8>+`c?u7#NLS)h9E}^+UwgpP{D?pQuu5;nP-D4c8IsDh-=EasExW z(2$&L79*-YQ_#pJ9G&<*+hQ389l5a2dfw?BTOKQmz$As5^MxxW4tIFyfxzbn2(Wb> zq9%JOM&zB;=0C3?Z2%73E6LP5so^(|vhp~g$6R;oaN^(!g5Xnxa)6XlyS^@D&F>Lc z*d^QGXxnbFLuVzgys1q++I^8lZCiHMx$PKiW%w7c`+?)$1>s{+N2=K@GIF4L za#qX$LM(|Yl2$zf3BR4?X2)-65OVcw8IJ6pAr?S<`YW#0-*zV|`jf%h;UjA23>6~0)ZoUX~4BU?MzaDE>Sb#%v;N237%W28p>jxNPf1vXhDU|s+N5Bd@O z?WkEa9VT+_Q=i#3G|(DQFEWZAjD4BmSNmxyvUMH2Q+bMPvg4E3kO6ZcDxGD}C0;Ef zvG>()?J&s1^eDV$wSQ>(AV@ytYRHDHm+cM!2EvRAS=a>DwU|(-_*K>2=&#XNJypy= z!PG)x>}2Bvt<*Ay;^i(fOOr)eETppKomLNWbmoXn+i`hxm?G*=wz&@;%k_K(RSTUq z0aB2V%@c?kiwp8YWyk}?;4m;j_ZVX`0=JY=C3G0Mqh)=JkTHzVMNuHV1kXiKwBA$Y z<{&Y^M7ouwut&(DDQ&+HW@av>+IMXaUyw+|r)GXO89H`;wSXCLRu{Gal|X908r8D- zRzfW#g4R1DmgAZ?FG)F|SEjA`^tFANCUrB|8-80baxteuuq?pB zMPE(pn5ot@nIqk%#L%IP86G*8N|T%>b1pn=pzw&sxS6euc0dNk*jx8f#?>C)unuai zx@%_}9usu!`>HsZB@(c3sFndMf_J-HZ?7v$E_3f<<09a&%zj`vcOYlPR0E&O2a-zi zz#wwQ2!RmYt(-6!%n(wE5}7XwnR6dyY)?h+HR!pWS=m`KrjGKQmBQ>`Rb}JN1<|ya zF>Tr*qReku(FOOs+d@oxD|1+zV^57j^4@&~DHtFzO=X|Nf?o!Q0l(DmVH&MyHuQ`^7ql#;?_43=QO;2BC_t@2ssxt zzb{JDc^uIt{d`3iENb7Ak2(mu-ZfZXqS~+5kO~`X@%JgaiV4M`4poRetR)m^_85pj zuIBahNT%EWI;UbsX{M+%OsEJC^sA{VTCz3*^)dIGA76n z72xwUwONuDq!R%ICP6LJ5o8XHNXgSb^|m_Do-6NJanISqnN2Jko23}rROK^JUe104 z|B0N>t3ocwN7LnP$+MIdFYOWhXrU#&oqs%I0*vtDSObzf;CR|G&$ynKx06KX!SKh5 z&(vAa_8C{IWa{ke-~ZQt#DPdgir>A2ViRZZzH7Zt0Y5jc$dG+^#cF4NX@&E^8rN>I zYq8Q>=pjdc{4jk}FHm!EB;Lda9{ny3gc1gaign8cG&YL;EN2U@|~O zae%O7{70)Hh}xS|S-7?7W*&1_j>W|_WBaMu#iR#}IqCw~5qyS?Fym*lljWx=Wn zL#y2w!E`6OE^rc`!ki&7b6&+OQEjMg(`B?SKI!Y=RBiDwyc%YuG%mR(RBvV zgkK(!6Inp72^Az`4i?N3k|oOSr_bgz@7Be$NOvi_U~2)pt<~#zRk0y13~!o{?=APR zY#DdlEP#9CxvYBB#on9IX~lfBwTXzVJKdhyN_U&D`V}5Li5tslT)$@vxwUkL&8#>o zxHi9#ibFkBstHB#O&#FV1gXzj{gjVte4Bs6HV#VTqt1+tu15 z{1`@-es|ajeZfX38$boQ6kJ!;yD{-Clfp(;nO3d)WX83{aK=u|pIvLM-Bra2pt7tV z`Br`xS8a7?Dw?l`>mZ!aaa;YqpAGtI|E}(aEU;pqEt5Z)98ZAx9#4*)gqQi~Tg%s~ z4rA*y)lc7qkndc6GT^nY1o`^--Kt z4541WL3iTZFM>i}7GNt_M=cjJle6K)pa-ndZ!ms-Ovl|t#UiyVJ>Q*zrRQ$7swMHt zh3-jYd`7GCs3PjxQlYf~uB5AQR@LxYup;M}H@0utb#JS`t=Q`CX1icZbsG+LFV!{V z@2cZk#BRK6r%rPh1snSUF;kvFGEl|z0}|ZYurn|^DMZ&z(Q zUwY8mc<2lZycE_=wkY^L&)5vgpmH%&6%Lzq`L#9;c~`7rG2U14794n0sd2a<@4mZ1 zx0Igqj-$S%&SeJDaaaf$kK1?n=DSzlo&3{C^=WoWeq?sMkVT{mo@nwgm%d;t%(>cW zxZfBBXWu$AZwlG5@xY~P?Qo_rwZ+9#Y@-6~oEbP5MAPx^wWpz33t0*!YMEzT+>Ly3 zqN%j#nT(h}xIrCWDz!vEl59C ziQ?PReQ2ac^aZ)RzPP@8&Q7K_UJsMmiG-fa>EgW9^H*E}erUXKwf1F@tA+KEUEMQo zmbv@9`!+8AdVWP{X%atRVG7HY(26yXQpH~7gx}VBESESFF~aH`m6u=vUC4G zi-f0NudjDs*#osa0059Na@;kB&l@UkX$u_O&gqy_@$LDaUw;cLZG+m(Wwx5)E+d||CK|o_EVlQzixp41An)E>4|KBc%i|%YZ7cI6 zE`p$bK+Otc77VIU_LR`w+iH+!=`1$Of=l&m+{C+H?tZDd7y=N`n~+-s_yOitA`L7I ze=?kjDdi~Q=W-^ua*~Yg6gOoSfANOGD z*o~Q^pV;_QvoT4TOD)9)Ie|?d)+xmmym8?b!w3q5nHSz`3#HO66PCjHY3pv)=p#Q>`0KCng^vv(CuqF|N_ zZri@;j2q3yw5E{%1OV?iGUMJxXBpNs%rdhpQiN3#Euc_4!aY|0%jrU4NsUxKhzDAv zQM=S|SyFp_vpj*tH%%kPM^w zSn9*umWv)ASA1o{1d}svic;x6mm4cCAbJM0gfqJ6mZz)SlBm6?zctlq> zb4Fy5GAtM~1^2vfFkm8({sZL$Z@DgLVXL6pb&u%CP$Dd}1MNYdSf{r;P18NY3ZB9l zx*kwic+&!I*arw}>-R@FBCL}+a@&=&O$bWwG`46PMuI?MH{2T^!O8T*$T+&BS+*Rr zPlmRd!w>u!^08fQ9~NC#5)SR6U>c#rj&L4ALNe76HVB1!N{t~QqvlUy=XTGErENRv zli@XRH?6%qr22Anlcb{>aYOy(0qqVhhqpl~&TUGwK9f4kyQ##yX)Wdrsxj}W9`lYW zGH+3ub+f9>+tvqu)XL0{Q=55R2S;`ISH*Z_PdBc;pIXQd6I?hj8$@%GPId5#U7W@F#V!r zBa%vStgY!~AyWi$D;F;6+Xro>4J&=pQ+BF)Xu(W3^q%sJ-eyimhd3iNtU5Cq&p4w4+4_d>$^}0Z^agZEd$PUPqmGMe2OQN;x7G}DaYgKL5GcuC}M&Vd4&|~|$ zu^>Z;5Hz*bS75JgI zveYiw8~+}$Tu>cBOXt(tvEg`tnzPwWKnH(40A)lNP3A@v8>6$?2#QW3d}Zc~5}I7) zKCvg)FTXplLBN6YXt*dQW#e?Ld?>0A+Q{CVA=u9~%y|OPPo`1(uG>Ao^s#1wU@H_eA_oi? z;4I8Mm{PW+e@=@#mU^*hLaxk&oh>19q0ro6FhuF;qEHSxvZgOZ zuB|MH;qIW=IX5aoQpPFA!_0z%2_Sdkk6WO(RIm=Je#-&Z?u z{ngKLNogYkN2h&*p^BI}O0-?dNp_<)mxZx~BopXk=3cY#)M(1JTgSC=5 z6PYaP{+Hv)vyTAuAaF9myt;`;w%`kUW55bx?s8RGI z!Tg3ya3fyjGN|8dD~1jv=as1HYiZbJ^fiR699rdgXFTUN5Zur`rUvDH5ORVIOp--8 zvW-o0mdj!}@(r3l)_SmSoPyYzltsaW&X$1y`v~%@Uy}G}lLKPSP;FWGFZx* z`jXaaR9P;}Z<)9!_q0&MF6R~^Xv*)o8lEFT#qzsB=ba$T=SI}J{`E#--?fdrw(J?* z6|&TeQd(le{qWn<)030Gee?D&H&?I!&QE{&n<(=;{OOmIzp>YUk8I@h z^iTg|;6Q8AGZ%U+i!pBLO9+!3)SO%H?e2BO&#T%*v7v+2gBsT(UEIx-Bsaa!Lq{6! z(fSp6sRoP>;adh;bn&AWW3K&UN_9JSWH8j{X2*TjVRN>Y!69bukz?j?-46kCG0!1j z?hr6{2$(xq1>2s3RS=#-z}z8VuI=4lkt%Wsm^%c_9RlWlwE=T@7aGFmhAFgF%-pXj zWKNkwn6UFCk)_Z#@{UD>!K@fs)>?o{VXKCh7()w~X4aeAZ!{e(Q_e)Jjde4E?)nI; z%Zikx)gLUi*Ji>Ka!v}C@CCOp6cm3S*%=&93j@70hn8hiPkKE(bP?DUti-c2m{|61 zhKBMn5Y-2+7*TWi|FieM-Erf_z9^j6^C_^BXD!>=>Xt0qGdo`A>@1Ds$>g|i z$;p{*p-HHl5lJurDOq#pe)qdj0N7+V0iyrOExYg{@u)!%00C4Ls_OTv(wGSLFRO%` zkzF?5(+o@se_>+?X4YttgwXcXJznk*VxNjcz)Bvmi0AQEuk03HHdCU(=cpl#YE;f! znkKQ5*Ea4g4Q85n2?2(`d?MqfgqhGZ_5+pyJI^q}3$JwFw9%dU-sHUn)t;s4U6z1Ck|qRfZxwG%V0>UoNqG z!fnqLF(HC?Ow-bYf7j-wxvAB(d<9SRVcIl4jYL`Yw1ZpVFAAj~u;LcEqzElU*m6LdSh zpMIC967Bc2AnOlLml&qh3pPn)DKn{-TETcWpPTH$QA>vCP<4W)fn$AYr2q;>2 ziIyyY3>-AB)2S|UzoREcY9XAEceT_&%qZ;X5t_zfIKg zW$|3wa?f0(Xh3TVbFbB{btkKmnZLM=oN$s~0$!)}}hN#+HcZ|1QVylHw+ArGJ z8z3fXN4W-NP1V`szrM7TEOTqc@B?g}%Gz!&X9cf%-z%=gIE=-|vueaRZl7@%H0F|g z4*+u~3~YtvIG2}V`DCAHR-{Z0Z~hbR@63MN7vp&U;?2>?+c(3^wU>8|`L9L!t*7Xo zSLBXr)XM-?z{{j~P2S|P66X1?*QhgSBkW{)=3hSmQ{dWczGCb@(Dl9Sl;4q6wiib`O!68=4Q(T+13QIouh;-<2A3@%_mM*>_XbP|f#Q8SO`` znC_d_AH%$6JWon4!fBs+Cf2s6#cssY)DZUYYjVtrXrF<$?;3>@dLRu5hE)-{YpW4D9nv>p#`2j^sRc z`xAHVsw3<=#K{<{h~?P*VR}jv}AWQTwyi!?bO1^X&ehCr4_(tfqBPqbbdp?rHrWzPBnDe?FVj8;Uq5;5O2^JG`DYUz!_zMQX z;~~Z0qE*niw6|$pGfTi8r6BC;@TQ)YbD2QwK_mXn(4r-Bmbs8#rcRsa=Dgur z=R*B0Z~Q4(VO!$4prvNIe!k3AZQ~iM?N}i;GBW|cQ7L(p&^{KQI%#R9N|6{;p!G)w zN_69!PzftYV>e|H#D{+w{x@hVD!4`!{1qEWzSliA9KBmO@u)mMPSpw&G**x z(WAXlZJO7s!G7^X)_Vo;G`1d#VC)X(@3qF}uHhSRzuE1Dm^-2W;Lq+FM+nW#_U{>~eIJTRsy|vuz&|K`W z`1f8<@KSLWfa7xd05~pNM{n6SX3KVwTDFYO@(`#jTf<@rh<#WRcO?){B|P0P2s>p{ zh$%~CeuSZ;ET10p5J)JW1^?vpp`L6F>*OI2PPU9|vK*SpGI?ZSIb_RkkS(H=tcgvs zF+`G0;E)6#Wnl?y{YWDL>204sV=WAxU`bGd8QegRMz}P6&oa)IP2Oe9TI{^Pu{e#E zVdRNy(1Mg)ou69`-}V~FBy92b-!JHwa;JvUb&dCaH0Ro{|7>+|IUQt3NC0mje0{i8hGhIIPySu8 z*(Fb{?`_Z%2kmWsinLlfClVsQkYQ=YR!^5b;rpivUeLN#EY+e{ZJ*C+$E_;{b1=f0 z)qf4!R0Y-NRbwzXIb%k$L^Fbk~7P{+%;orCBAg%1TaXu9KYA zzis{m4ABa`F`LrcgjYRzaw9E@|7acqRqLKtwF9Gq^vQb0GI%`PdWvoRuZAjyq9*C*M=qm-@kpP77JX%ot|`rD%2uL zzu?{a$O&^(u#~7>cybHVumNxgE7AI>IXi$a&gap#bQFn&YMW95`plKj{ICWphrgW*AE zlNZ(>(|F?caQOWBQ2T-CnBR7+a^GdZ9UJ#d*$Lc{%2w=tjid+))~qvkY)o93l78u> z><|4Nx>sdqmwvpwJSPPeG-D+b+ArnnLVq`)D~!g>mxbTz%bDQ>H&5xEZTvwrjejS;+RP`|65mIs*+Le5i0B_kQVQRCRieE*R+eY|K5?1vEX+S7BVL(r*U(8Td$|SH=%{nu{57KsBc=M1;O2Y z!Z(W!iG?O*HP-(n!K{Hv?6EOOePI!^Tb8O_GtBsJNt)6j7n6OK4?c|amCG+|ysui^ z|55MIedl-F#CrT>_Keb-bpXLK(H+5S@$Un>H`R*<=1rSgh~x-@1Xw%+Nz1v+lk4m4 zeUL)Dk^QDi}w}f$TWNrL?G@9i^Zf9})eKZ<0@-AnqgQn@Wgwi_SyxrgR zXF1>sS#|d?P_y&SyiOm2eYbyTZ!?DE9WRYsHoqiN-`>vdJ3vNXo_Ph_FH@GQI$9&P zv5~Vpj6K`Dy*V?POWunN)hN3)0TcRQuj6%C8k1;B^9hrCgc8BiRM|IG5z}&ZWm<}4 zGGW5^ntsP)&sWN*5}~@zhDmyCW&vMM8am5nt5zD%+zEoshprOdwijgH2S z$Zt$=Gi@28Hk_=X9a)!(j!wzjrTnl?_I`l&l8OoIi{ad6sPN@1 zMp#oNZ!|(w{aTdd69|+w5LN zzkx`21qd!>wcYBCDT${0QGa?qJcLwKI7WP_rGMkKo3`CJ6*heA=$0la+ypJwh6LCJ%Xr=y^GzWe4m7+XhGIrXQHgr9OFu)Rp3BSsqrUE5R}YDA-wljFJz zI^I9=hpPMl7&H*5RMI2`jJsJ2ygvvlz`#Ck%}USwdbdJUiCcC)1J2$^2uhl&dXaA2 z#3MAb+X%#0k~haEKfWPTn#XcVZ`hvtb5VE2^8zknCR%qK`bF)_4d)fcnscw-b$asI zuQhJi{isK2>rvQj0D;nvHXAt5tEX*AM_WNZ3ltaqdReGWwwyk;iZ1q0de{T$V5{k0 z+t9r}m)^D0I#=KZ3e&Zg*Rw#2>OU4jbgWSR_deaquU7@>RIBJy8U19Ibf83dYhn)@ z?uZ$RG{g!j;W|U2^~p5wU_5`(7k}wpCRvg%IRG7cxWNG&t~CIyyl4gUc*B0BW4*iQ z0P(dhtL0=LOn}xxpGX47sqU+o^H1%OmL9NB+1Grd8O@O*DmFtqs2iMVczc9O!ahMF zBB*P=?=xju?NegKfaU?)a?^jer#`$Np|O9D`}g+%8<*Zf-ve*lT_Ao=^AnGNbIrsHK`zrcqXD``wi zI-JbvfRYn5dU8hr?w5?IlCf36^C4VOCB{zMBgr+n>UztS&jK`yDZMAp54AO^I(s$L zi)%PLRsQ+waB&CWR(^HqxwvghnEM&<*6iz#ncDV(>1e7I zd*R3ZSUhSwt92+D_NTJ@8g@nGkS+vcWdz=^`;JfOc)JS(Iitl~-GVBec@-t-Vs=(p zvxMgNgzAhnn{;30F%#*%`ixq<#%4;S*J|Q*;?qbTJ>Z~)k0*1<)ye+t3@|6ByVUmwSUzV zo?U&WL{?cw#q4@oJ#Ef{CA8c=Eiu3-=@_L)wb+hGOqPn|>pA@X#Id6^%4 z_m+HsUc)QgxNEIWpIX>0(OPRO3*IVUZBflnUZb7ht=<^@b|+ZdKJm78P0-N}!*p*J zrL0fst%byt2~bQIEZXMG_%)t|h0$3{_mDgRb{qa7Z2IU$|xDo{|^a-UA zRI+w_HoK8a?`L;%4JZJpzDao>bR?{I7&N4O=Vi|}F0pzKfzPbdbsquNWzl^UtU4Rr z8^3SM4~!Ks_s7l?pnre+cXsEVUhMV&CgvOGQ+m6+20w%TgGS|S{-qk%cY?NI8PO)# z1nAi#quT<4Cb5?oR^zSfs{Ys!*{bT>>3uq@29M8IG;KDk8l#;2KmYGHf57r3lxRvt zDfc50!~YCsXeFXbnH=8JEM4~<2L}fS-@SUJ|NmS4{|ARJ_5U9{fBD_ZgFiff`Q3}} zUL77DzIgSAgTv?wlt0zp zFG25y+5=WeCJZtrIKyUZVxVEAx#;}c?ZNQ7SHtJ~>=Vt3ly%h^L?YhLc%GD8Spma{ z3&y2-%g3&j_sw81_+}RQ@|w^hVV_EtLnh+z=072b<@WhEH%T79CdZX5d3IqAP=LRI z3$kzQWwAO%X`XW(S7%Pxb{NXA5mN$<+gINn44=OmYRn)zoZ+m~t-5+5zA;f)WPWI! z6Yz~XnN^P_f>*_B(p|}*^qZd3l1+d@SO52bbWAz?qUUEO{Vys**r!y)pE`a#OH7bT zkye7H3uDmIYRUyLYqj1lY}+!)Csj(t{KMZ6%~|kz25_&$xJ@usZ4J!K-9FbXMN^j5 z>5#l&`O*35&o4%;KhG&m?eUm+8mZJ*F5)DgXq?}a3GmXupbSrcr#@)!;Ojqi-;XpF zYfFXj>vL)}Mz@L^piO2*l<@^4ts~TGmrD&FOmm}Y%J}*Hud7o=(4^j)L%+}l!lMY^?lTA0>HkZKNUQKHd!7Q zJju1Y$^?GPYMdpd1QnD?Ak3O?_85*ih+o!Lr`n{C>4qfbT24CrqtiuoLU^u=|pC;f!1*pa9O2TX!gB`%=+V zCNb0IER$r)?+CvxS#Hj08z`?v>Ac3Gg3ZS1%1581JCVo+$OQ1gsyBmod>**}#dGM4j%Xn|q~*W< zH#ry{zVzQN`)*!+A@Qy0Y{jsvLBc|Fy*NwqGnP-3(foYXPhF_Vpy@}8E&EZHLMeC7 ztc1|df36$8Fzt>^vo)G3_{PTft=J(o^LI&_IvX7`l1y@7Xd2%)znxW*Jo!f5vc<;T zmC%hr!p*;I%w3Y(Q1z_fxss#%>cc2$Vy13jr>=ja|228`l#aP5f4`$5fA-gvHz*uN zL0`y%tknBW&GAeXb)KxGKR+itBG>4_s`{9y}LWf2#j5fImF*&qG*V5sYDz z7}d?A2^t?ZsG)Qs5nK>z6}2zt1i}<}4lQ2k@n6Bxyn?o`>Wt4`{`J75iUs4_Suxhp zG&DUAPPPwDq+M>xk7j)a2=Ew!2^fv0<_m;Ovi>w+U-qg~^CRj_XJwy?j(~_}q51uL zFE_1?$_Zn{AFMdn>e%@2GaEv0@P(|7=AVj|V=Kwa0T%=P`o{Hnce(Bg7jgFS$S6kw zV!C7!GbQJ)6EOUE4u&1O2j@u7haWSgu(Gle%7oX!a)~5qZ!*|tR7!OT{JJ-c6(9A? zd|Ymle`2io0UrE;i~7z(srhPlYSH9ZLv&F_Xp&dVej<8ZvbnHC6Mkwj*GbZse>J6& zL@A?QfOZn5`|0vnKB1wE-HjjhvfW;%!SjL)5!5r%FzERTfoA zO1>zuGggWu>Sfj=4==GpatGCE?)*96OZYR#ACdrX*&vM0w}(bxJd9Jh&;K*;|C5oR z1=E`HX1srX|KIbM-#&lw-K#nO-@(i0uh9SZ5M_7#e~T;c3-SK72llyqeA8*XRm& z$nF37Q?6e7QN|ymnV4;t-lQ5ex=~48>|#sg1x)oG6j=@AO{0iOEgJNfR?|XG!7-=h zT_5NSZ9d()__vme^Fp&9$AVVnl#ArI@I2+C<`WH;#!526bHdc(wd=zic1}NVz)eOZ zSgbwh-s_Gz5HpQ*LM>nzwTM8*hh1wAeq$ZDu$wsVjn3$Eqzwk>X)P4JHUPVJYsj@L z;noJFMmjh42=whBe3-4142Flt&j zE^P=ZZ679W;8pY*c(kFSsOM;EvW3o*fZP{s2wIaR2hB>7}WPR}89ui2z}zuN&}`c-gInshGdyEr*5Yq!i?hSlxi@a2o)!Ro-W zN?i)u6h1InbKsHpaCdSzJn(rzn`P#omaHTiTb4+r7(mn|mE@}BPP$Tu;T2r!d{jSN zt>p&-H~Q)3ZeMJJe$AGPZZh0H@>`hP!x@_X>B%Ov+dwYg?XdX<^7$^q)ELOsxIU}z zQiN}z{EEvlEUv<_xH`+?!}2UXDAVFPT#MUdTl~Cyi#xB6lV!JHIP2ne7=Kse{9T5A zF_eHYgo1Gi62{fpf5Z5Hx5ohdaFS&+Jise60i(1+X=SDbrPbFbtyWA3f?yW~FX)DW{3H82#p-wI;PNL4S#_BKk{w1^p%HFKHI^m+Yv&gw)wY=TuNX zXZZ3M0fC==DGAu_?^S)fY{DGD2U4itSjmLO87K;t$2OU*#E}p=O=b=5%6m+;RUT0by8nN)Jc5>byCzxn+0{!oz+SEwTWP? z)CCDwf6ny2aetx{O{G58L0NP*sf4C_B(5OBtLKM25)$0d-<6L>HmY^ z*_-?}5j@XWzBV**BV4MInrR4L(+&c$>zR!237fr~+)|Oyamplls`U7e=NIq){*OJ- z=|}cIXdaVy@AZFm&`{P>eNC!dvU0CE(UoKp_MSO{`ty|5{9?Yn2=n4mhl2y5bVEu> zZW<5T5Q4b}JSqE4wldlo6rMV-pUP){eZ9Xw{EyX9c-ggJa(cQ9$hc>UPVBmaP%*84 zK~Pa#HOZ_angf|D6>-Y z?vZSd$UTy|Md=$2yCh$es}~3IYL8q!&)|QjhX>j7SNbon;;Y`zvU^DAf=xV_0R3Fj zu%=y)Zvwx&q?3^jCv?ML_BWDB9UTy~gfGuVa|b0Y3QbyUZ!f@;tv9#530W)djRu=( z$Eq!WL}3=!p)_FQpvc`akz7Mx^|$f+8fJ}o-SQZ^)>ANa4TAxiFT-Gf?s4s`8A>p8 zZLvJEcmNDt!_YN-6X}&;L{5>1y2$Q5IM$6*)$2^xbL<2!sh;l-7lF|)twV%xHFV!yH3(DgD*fN zV<0L9qWTI3qGBLwvtS_V&IY3DNS1fYK%Rb(dTZ-&7?|!Um!$JzG_f{PV8(7My$UP$iHdOU__a|KcX&~d9xo`-_n^<0D?qqWKIL=?^ zzL&G!=;SA_q2ZBWCY${vXerhR!RGqy^khih=M5!G>pe99f^+gazv#;F$02Y{-fv`~~8}Tpm8i#o;QKhMT`I z{5+S1J8MG(Zk3P=!fjj*23-t>TnhFBr?@Wzw~`~xmQ0OSN2%5+8&9q*la!^fu6n0> z1mpthJl7gn+tI5G!UNeRAaKQj*^WSWH6s^8t3MoI3bNi=OH1m7iTN$spI>V-A|0alOdet*pFj zWHqk)7^st~P11Y}l+vU0j^4}qOmg*SBe}*br^_qQ`l;PepnJRxtxvK!uY-rubqi#! ze!bgI;2L&ZZ;iAS-4D8;`vKh#=zf?h=zf?l%Nh!PV${I1e_Ldzc=qdpXFs0(c=pc~ zJp1R%a%aD%xrpboN<*Ej$L-o3URGV>^SfEwJ#BQ<)xmamI@{c=qgBuRdY)EwSdL1X zIz>jPFgOA~v{X&m84^rZX(`E5*% z5d=D}g^|`5OV}}{@Q6b1#d<5+aOJqd@Sze+OTe@QOiRGD1WZfV<+KEMw4*M|98KGb zeAH$2_{X|;0a1{RSy?M(!s}4bC3I??9*3qLv%J32d0urW%u3YLGRY_Ns<8!)bsbM3 zT7uuxvD`e7ZF)U)K@|yABvg@5MM4$nD^ijC9vNMhl2P08&gg5N_j+6+v?GTcY>IsDB2k87?D;0vZR*qlN-bdbk3WVzmA!2M#`#?0&V zA76a>M1+0uXP$=IzY#P6?`&D66@L`Z`9f=L?U~7=2?I_2Tr)PS+`6( zSDI5Q;#o!qQu76|*mPh+O-4XX+=vw1Jl7lvgRl*|Sek zTB+R(S*$?wmE@Bl1n%c?t4Z~Kw{Oez3o-=qDjPGgO$IyfcZu0sRyz5JmSdm|kb3O} z$YaQhpa8>p!|wOBHdN3=vjScxDIBRhX40$8Q!+_zS#G`H56bArl^GFsuGT^1I{)x+ zc+hsauH`T_w8 z&RUEFLCF`(BSU&cP;vw%_ZO5_oi9Ua)jckhRtQS&uex1EssMBzvYJ_`8(qq+r!7Xm z`Dd+30}F&6Jx@W8p0A)s4?TL#f*!pc_2`kBlK-3v+MQr!%W~J*B>b~4B>~(0y_yHe z0k2Y_eq$xm_u#ZBSRQN07Im=QFuywT;o__x)~e@g$cPvI*=Y^>$(4TKs!mOt$!yb> zI->dmN9z&RkRVSy^_@9&^P}zkgl@c(Mjh3!4&caV@^0~ZG?BR^JQ z{bH><_+}#j6^De)NIxc+br1AZ(5INQlAY029X16OWfGqB7H3IL&h0bX_j`hm@R;X0 zi^?A%$8Iy8w%%pmamDl#yVnVFTo`4vL+M79*;f%&W?w;-8CB+HL6v!DRpx$WC-iVS zUyyM1=Tu^~zTVLp=29Q)pvyX&R6&_CIJIlXvg+e?8k!K-<>7 zm~St_ym++r;HWd*kka9swvD#aVD2@qlldlF8SMTX{y18BOt1~ZEJ14|=#Wu^^`ia>#!yWj#|1e(H}Sq6 z^M>V98eGGyN%vLq@EP8(ed5dp2eq5( z7S;0_(AD{LkLenmPwrD*TB2GFyI)q9bTZO`#5`;T^_+%gVU6$244XTJl1}t2{fmQq zPI7yxM<)b8yB{Eg+=q!wyD+mlhJVyp?sj3p;3`AKxN__+B;&(s0*|c?mWESXy})4k z_yE6mb&aJ`2U>-}W1o#oT$W3`ghT~9L%SE=cibS3Hh5Cyz5bkFM`~_O;B<1)ST^1} zYiuL5%^28=fxQ?ig`rZ;aqa9a7{S2a#q!9ar7*A;1AF}irB&z4P+E153#AnX_WG-C zH?WtvDbC1(5v7?n8y1~9Jqp33`ZFxon8PI+R2*y`RR@hfH2zSWqBwPq3&klKe`x$c zLE~?6s3@&;L1~53s;{8&*DQ}rR0X{v$t8;_!A3Vp@iX}Jf=$7bL;~cSmRl{85RUs^ zi!W^MZ`l2kNvGWzquL+^(RQ7rVFq7-NX9sLjDzMI;b85W%J1=61Hlpempuq?cP0bGA|Af;=LiWrW&V7L{86dUlJO} zf=Nln_g1Lvseji%*kfjLUt)c{3I?I(t{u5qUSP92xElF-?7&Nt-1-QDmyEpwaHLVU zE*#soJ+UUXZQGb&VxwbCY}?kvww*~Pb~3Sjdp?|dzW+aU>!hkzb-mg3c2)PTcdZ9& z?ZY0s>tPPT(pUTnCpqV0lNi$U2&MO*Y5*N-IccY2)S}urM(kRRE1MzRD;vmKjSS$+ z?ZF1}#3{S6;jnq8;hTW0%EPxXJ4x;C)sI=m|K=myi=%<|zpWsJH8xKd{8%;_f*nz(6#6Me5z&b(HmB zYGVVnj@KksINMu#WL7w#OgP}7Oxk#&0=1^rXsVTT@|*gF=x?6HF#FF6uUG!?Myg_z z(kS#Oe2)a;jlAolQ+hoAf3FnTi{%qg4!L@PsjY>F;+V!pu*s5#gyJw!GL?LO0pJm{ zb$(Q6jkY0#3i*A#AtUa=XW0F8b8+bi7-n4?K^F9S)h|vYrxrr40H$xYnd6hGGrcGSkKSM zyWsqa;F2}bd=+DS2mssop%YcaaqXou$=Bf~rPv_TdpMTxhJp9pg7F@b9l8|EQ!Rx@ zR%&ng#HwAalFt>Y`O|#t&`sjmXm4X7l=_UyS?1jf2{)di#UR=C$@ZXRbGJNgTRlRz z6+JW28-Xxl;N36d3;CT~TwIzYH$U`G5H2O~Gkx8)Zkc2#6W6*uzpmW|a}y&Oehd*F zOy1BrQ)9NfF9r~_`E&C)eLg*0PUmjMXuKZ!u|lLGbymC`%^{-q(Ft|9rDB0_xCx=o zzT;$1c2UK>3kkkcq`r|zx3NFjbUcA7euFr zv<@rkVSlal_@rnBwsivK3#-rYqaF+q%ZYKwo>6wJJPDK@^zV6!kZTw!PfD_{5^{

GXc?~d`bpd>Z!)_BoSZJH1 zP+DlW!<&Fjj{l(0A)^8@0ega%shm#DZ#pHymiaOWaiuA4F+GGt&PM&BIoA{Up?+5j z6@5JSsu`c|Tu;Jw)NciR<5xC188Mj!Jn!`H6T-ku>I75w% zaZ-BnnofOiWhyVe1D^`a^#1Dm2j5(<|10Ii{6AA3DHHXL(7Y+#rO2dn4h(lm#*Oer_1aNmAV>Rt?{=3;YJQAr2t5)rfu3;osY0+ zFzCw40E<-6UwU-ftSlAgw4{=f!xxig zGGO+2HLMBVgcgk8T%BDt zy^*XPq2r@d<`xChrqHU)rTyR#z3M;W!q4l`>$4$H>ZgP>Kcu012oXPKY7ak=q5Pj? zceYmqYoRHFAS|Z<5SA_U@{TQ~NWUsy8i-+M^dJ)%yy2*HODHx&QJ?{6mSuTH_I`-d4VC!dWl`{Yk&?cdjqy;%L;i$; zT%1Dzo!gwnWyo``u#qbOa2_IyJ$+nCQQ5Gvzphzp(CMTut8i# ztkU}_jWPU0!3W4x7R@BbB=xL0*NTL@e!X@?$-IaTG4)Jtt5$ghwnKQt-)ef?SE#l! zX6JCR!7!H6on)j zalc7G`~vegGD7`!b#bW_^NJA&WjQ2#X9eU?KX_bz9L8BvB$i{7l9!jZQYFm}Bm-iH;RIuiDR`Ag=YuZ-1h`g+{~hk8OeKrZ33#;Kjr^Z59J zsj^$J8LKnLQ>tX(aJ=dtmK5AJXBNYl&Fy$C1}F~}y^U4JH-D$LvxASS5V)OtE?10z zn}hRa@2|EBlC=d0?^fiRlZw6SO5ExioO$EKTnaaJsk3)Y*?Q(m4y+rWGg?&!Te+^! z9s`=%P~>ZX#>VPCJIYTpQIvZ@9HFQ1Lz@O6Do!UK8G8c=;$Aq9>e}ebsgg)B71nsA zM>_5d@Vr{$`pd3Eb)o^5Z39w=n5LQdqs-|3X-E)D5={H9_1L%DaS#X6V+eMl~qx}>#o%9(8dCqJ~|MUPmJ>t_%~eN2m1*XBc{(#BR&I=L?sNpGpm)jAO2e>G!Uc@iOX1IltO*p zTn+^nGb0V)IesvSmDRY&>yQd4Lpp zPTmQ<_=QfYJ3;<#joABcjUh&J}twCyrT@Rm$G%}|KLB|@y!J^=F-x8^j}?IR)caKNSNt?=T% z^CCCu{y|k8Spb-y65bJEL8Ag5t*+rg5cHAYx>K!{hc zt5ANdjv&c~z!iFE(VDK#8>NM~pwsu3t8_uVEYN7kFC2x%{s_#Zd4GASf0$UtPi${G z-ODy{PZWI{F9LJmUND?B8ra$^zMzj?fVYS7(4H2RB^D?|@|*yxk3U3X7rKyR7xKal zuVGXjJ!zuf=LgRXQ5X|HqWgZz4U0wI?O0B2K}CY0*J#7k1-B>=C`OS%D!AFU+A@P{ zEk7U`RfjX}$ByJNxC9mA6Krf77UL5HwPZjBwdiFJ=Eaej4tJR);a(2av>G}7{{A)>~+%l05kNwycNKxh^_)w$i5-?F_7Kp|| zA79cmLq`8jHVI4dP!dWZQEhuatL5|UpPCqnpeOrigw?F0Fp>C< z9X~8aXP3G2Wj*AZ7~8$;>apPG(M-?B^U3G?t>CBS=Zc`<>)9uvz=g4~F*5~)The=#`JQp|%U~RQ`4mvuYH} zJJQ6iB>3+Mg(DMk?s7^jY~#bX3GHP;9l@r)cl&YuWgeh>H{@{N+Ush2!Q zZ)#}^)LmMBb6-0iJMQ8sFQA>Go=nG{mrk#ePs6>w7G03m9__L<6D}`G7`yaT}WLY zu(Zk3@cDA6ZJC$V^6w|tKU))>`91oVPJeardy z*0tNzB{%g>)%X6p{=(;(!VgP?)5B7KO`FnX7-+GEQOOvo-yM{@+RDimv1w-o;lE5v z1SB4p!Zi|oM)kHb;H#scsjr$8LY)gob8~(+_cc4axv3Gq0i1ry@ZefUI}zoKsn5g4 zKa}gJ2A(+}y4%-iP}z4x4=@;t$)4=AG}O=7uWaYbR8vkn@(0V z$we;O{i+BxT6A4XxoPiW=j=JP=X5!8sF=uHI4!MuW3VJsYyA9P;}z!SP~qc;=wDOY zML(WrwYGWIczyha(Ub-+a9EtFg!@JiB~xqBZZGp=J+&E5&;e=D2MjUONMXt!gkg(5 z44O*gA^ccj`aq-f@wBVR7YSg?@V#OQ&2j>}A}0VD-?_w-3{jQ>7yP6!@DUciMqJ`3 z1Td}TGF-rhm0f=vy`gPM&&tDVc_56N&9B-4_p4Es8qBp+l5*ZlHB6!C1(X=8yhxLP zPq-K+nM^HGiB_c8Dm9s`C_&<15D6&ubkG*Y$>5b2@j{2H%_ECbBIevEFrGktXGf84 zZ7YtKT~kI$FDWHqMLc}@Y8cw~@izsyW{|K>0aL)qN4Q+jDEoh;fOzXS^>|sY`hii( zmfMNqr6%xN< zCy*Hg6A+3d=%J!#LN6wm8JXACf__Y@>m2{xlB7`S{P9-!X({#F`NM1H+4an2?L$tC zVtzQBBU)_`Y^54hu z%1uASjKTha6&5)!1JrD2s$bxM6xz+G7$%xEd3*~a&O5gu^NTYlCeC}rTrT0k zS=7gyQ$!QwWzjdC3PlR(dxu`n=corCQ78b8XTy{UsP>3^}aR=+g{+bZje~diQ;4d$M+u zpO9KkB<9)>ssciquqa2AYk`55V6z1<076wvho|`@f>1o2tY;XUw9=R0Hp#2+R%EEWaAH zQnrvQ*b=N7HP%)Li37~dG9Ao~1stH0pM5mbGLms^YRXh2vlWzx^-V!G2z4bbxPwAI z!YIgJQ|`C-z zAY%kIErE&n-z)L__pc-9#1usL+4Y%=oVtyq4!+Sb7omyL@s6)xt30R0FTj+84eSH! zm6JcwxI?8Uq)I*pjjT5)KES*)d?U^6E1$z#@r`qK3K>>2qG|8)3t9hrRBVT!Uyocy zz9o*b%RQ4o_dJb5uuQs~QUnqSCw! zTR1UetNNHYh}XUevoExY5Dc zwt&1GROnK(NomvKPNHe?111PbN_}+HD=9a#bE5Nm`-S>0 zvfO*k7iM+xlujyX1HadKmKbS+hA|PJtv?=!L)Wj?*AYnUJ=zRW$tx@>&|aG*Ebmy} zRjI8MB6fOi@N=pDmSorV1a!b$2I)U!2|%5~#H|PW}gGQaUN@8QtcT zG1eytR7m>7m*=GuazP+j5VcC58OpYgl}eifmqV31pjM`ukg>Jw#b2j3;BD&zFTa6v zqT5%DVN4?X3R=ahlm_?T7Y> zO&Yi?TYJ&PM^kaM?z&rCBT=d_$}*K~z-e*yUN%gUc-pjN_L-o!WMrl3eG1dPYr`I< z8D1Re?(NZf#Ng2y=I%WOaQEi*ZVl6_nAZN~Nxx)AE5J}~=RA%KR#Lzde2P*bT85O* z(^Pqi0=)1a%v_gzW7ZGV2Oy?;HjX>^6|gZM3kE!-;&fYWP1v(owY6sR{e_3W>ZrM~#?SsRj5uMzw!g86eo2`B7LmKdAvdXU zwouBEcIo=}@v2S^`_g#HVkR4U4)a;Yiyv6<93ev?Ef*Ffb8rK7s&$lZis&xlaX(BE|3J3?zC;C5mu-@LsSnJx2eK%-=n z0VBb_GK(p!!kV1@Vv6Qf>!so|Rd$2C;j`1g$K34nOO1o6GQ#D~#SPFHfF!&2gn2M& z*pU*2HBYMW)m_#a|4uhR&!u$taR{b&pD4pXFn<3Iq8_3h&II-e-onzY8=;Y{W2Cv! z?TavdAW+UkV@H(*sfT#`?{X$^X{9~36jF2}W#aw7q|C;#%Cxv0%gdCYt-Oqxf=Cb9 z3+7yHxZh9?D!*6HZnXD)7Tl?O1pUadWWV663R>O9R;8x}mb^)!pJ{QvWr27;Y(^8q zs6(0|;j`oS;3ajHn0*Zp?%}%=H~X$} zM~-a?pZh#8l>U+j0!&-FMTw%x7u%UJAk5HcoaDZleoC+SUUbK*liFj`#}zePX155( zME^!wK*$w!DZ;5}cKZ#}?3Nw!7vGA>?S8pxqV?1Sq@rzcVPMMO4Xbj-w%%fj^EMj{ z$_k9Ofw{fw2cf&j9s-Et;Pc*S#95KQ`<}qb)=b-aC=FlF(od3UysFkvJJX~WuT`*+ z@5?+>)Bx$n4z$b5E_fNd1v2{fR!%Vs9Y16|+1R0l;g6S@c;7(Kc^Nb3>tzQ)H)D+x zG>fZ%m!GKk#31XO^9c^#XAM8c(9829ZP*YVAf<1H7oXJywy>6mI^yjxXWUxU-!j*h zZyfA&C%XFV5Fa^PIbJwBs#jVMj@EDO-aUhlU-tJ;Y8mnu{4 zD;pSw805`h*VYj`-0f{0YOyjG^hd4co(J3Mrz%O&NCcfP}Gk#^m6k=zM(x zWpFgs>2Wm9PU%CJIBowLYJo{@z-DX@U43*W8Yt{t(kT2*YVSOMMeHfSaBgLz)f>mZ zSH=kbgUVzh7i?t2p%64=mhuBuD-^N9H^6EIKEZ$TZPE|VRMKT3l5hB{y2OCBHohw_ ztTvlY7}0V-EurrQE9O~M>dWB8k4b=VDx(n%76PQ#+2zk}~S;U!Qbys41aG^T}xwBP1r<4*O-g z%V2*M{GdzWos;uQ-8w2oH#2ANl4BjNWL6@hq~%avbR825B4@~yC#a^L_6<-1MH6G@ zb}j6iyt2=l><#yn#`s&t_)i>e0SzAly12YY_2?{Q z{Cb5X{8B&ky8( zMd&il0ct}q4Y)1hhu%3xL7waYwd6a;TTyW@uA(r}>?Um`>T1Ja9$MjS3|Ez+ zu*@(2_^|U6Nw1&NbW{SD>`duDU2{7(t^+Bl-Pvi!(vsby4~n*}E8J|<2b;`dV%EN< zY{0)S7_uTzk^X>8bQ;e7N}Tv2me49mO@4D?X13-!B^6-2xPE)yfYK&FnSgSS2piNo zOZel$k&WRP>GS3H4o>G1Jx=GrDLs_Ke{Gj=lgG-wMYUCCkdUnEAEvu%+g3>fXU7?{wj4-7}qZf=s-D3B$EWCw! zg1g*AWSS&qaKM@P*j;-QaDpPoARdDzadNejd2{`bWpC+`f^g~Q#Pt^|$@#T483qkP zrABWTzf(3G5&Ar$Gs`b2GxHjXaW>1bPedjs^ri>&4m*6&uD2v9Q*;_rhYT4rrc;0F zDAAkLMd;J)w3vibWW$%{bJzd0XtX2@H}M6T!T6jLELSO%vX~`if;VLcuGlc^#`JX% zB{1Bmzl26=hI@pB|0@8Wx837R8MUlp=b#+W8QHm;P$LJ!EN@@)9!R*U=tRn3^2vfD zQRqZEDqu*tss3pX{L7B6$pbBdcMVPPQJq2f9dj8dRT7P4S~*`ZiHg>$Dcrk?69?m&WV%J4Tl?lPC7^d&t*O@ zk;+Icc2hbiD<%q(8y{OD3V3&*3xf;44L_YT4KoV9IzN4Jhn5`oi#XcJov~nLLRFA{ z<@XG+ZN3FPHg6qKkVf*qHdv31s$u-~Z)ANW$okFX6&Li*^V4YlJM#dx7uvCkKOD~w zOO@EQddUfTnmktuUhczfjH5}ov;0jEvuHxHAPk^Q5I`U!mp+>Gi*7W@8jjUmfL-R?{YH|dWyL(xS$LOOj)~~^(kBHv9lAxfK^J4^yiAYiW4MhycmG~sO zKbk0w9JXgF3ydhapZ%%>?22@T(`=(n252~=O3_U1md6+V$$L%*5u}L~T3puyN(vh+>b2ICO{n zyqd6fqEoN3v;3OY#&#m7U2aQrc?IAp^l*N^*qQn)gg(~;KHI18=C-&)!tY|GK4nSX z6O;2kNa4u?)=}%<2_Z#eyx>Cd@^4$7SzR{WVr?xMkOBD;K#>3Jz_JJ<8uK58R4Cqa zC0Wa-Nf_!nF~8_BuE-3ES11^8in&K9%DR6yNy}%y73#XbUzs6xOV9TTo7P2yQ9+17 zV&!{Obz;}6_B*vI~Vulw(_i3pql0!wZo!R3Lve_#oY)f!Ou z7h0%AxzRo<(_@B-k-0ILJ!+kL!FtxVj4ys+1+o?66u6DHzF=}`f0n_Q0WS9ULb>$` zgOA)(3E?nq)CnqMzU0_>jPX^ATVnpP{35DUGHol*VA%ucD|`8#au5?9W8pSgzlilf zlL(KtUW{f%V!aS)jz?ob?Sh+(#7{$2kdkL5m92LX;HhfbMn}^s&ikFotEm5=k6Gc$ zqf?gt4lL6ojs!%6%eX;q?VLt$EqxMNV>mefq=%~Lep)6hlNn7;KzI1ipIc;qM>oCq zf8=<-X|q||xWrvO49QVSy%+#`b}R>;dxit_?%wZ37i%&+( zv-NJ|*!+^L6VXNiR*Bst1@gQ!qDMBCti7aAH6_t<)XLPwwm-2D9K>b?U|cPdDJS`( zN}860V#1d%%{U>U#P_NOhnGymQ7iE!bYP8;h8U$9iF%1jMV8S*KK-LvQo=4~_>sj# z%KB-=V8M`nI+Li~Jwz`+D9TD%0}vmvt)6eXc)}pB(rl%X!_}q+;!9{>3#HFlaH8!w z(sYhKmv9j9&I_;uInqK_sIT>pkyw6jNOksBUBgH2lH>?weoc~^y>h`6T2croEKvK> zZ<^W)WjOZYHfk|D5_6)87pQ%Ee{)lvDM213JIc|2C2hG%1-9-D$>z=@ON0j8Inf{!^WZjg*|xhvlPDFT;Pn z&p|Ti^Y&WvHa4iwP~hXmQj=^FjVggxS?S9Mv-}zGJYq5F)L-0mX;ms(i54A{XmTM5 z2NYwlb~%3h)|!rshxIx^k46o8=>vak9H`75GQ6$~RHZLktkkT_TV*!vDth8~HlGow zf9ot)@|7O2kJ^W1U_lgTan|82Qe-ZN0*3@{^}0^&`F!NP+*H2_-y|VYi#9muoJq;t zO@^RfbouKG5DdN!_iYRqZ)i$M-%WBdX)(&;Z3L?rbTMfO*znKxd8unQ3~K6Y5~RS{ zb_{C5zax@&WW;O}_7pd|jD>A?RP>B_C23lFjX7mNqX`5TF)Q+hPyS&HP*hxKir0qI zQ?ah_dSN1l4CcQSMuJai(9KA_U)n)m-$p0%7&j7CZveR5dy&s_sb+>n(1VJbBDTYt zZk1_s*F@C+{`qM)UpH;3sG&~F+p*h`sV~vva+mBl_l|C*zDB-#KeYPA-rUG)BjIo= zp8a@ud)WnxU?S9PkZm-!M?Q({z$a|8|FKhr0Vp7cS{&KBr;=& zsd{#$+KCMC(!pk>@j>SsOQ3$8RF0$O)S4~RL*&m-uvue$tVghXrlo;B;D3G|itE(u zxMu|RuGK-fF=AG2etbDoj_@48Z!}uT&RA%Bz{}I*kemJHOe5M;E|r{Bwnq?p*lRQa`ExMXFz*#y}>$RiuKy}gMb3L}lN%$O&qyi9D} z$}DEQpkL}YCdd}$wRJNzr4-r+0y!uU)ozet3HkblWKRT|RN-{zGS8MUKnAr?m)L{Yg*9n-2-tRhpVGX{SGeCkep~F4UU&CA=jFBL z#Ix&K_TwbCJ^i@v9Dpi~@#H2<@NdFQ^aMKvlu8%5bO9@Q=}8%?1`79y0NNha?D+d4 zif31ydcKU(76WoJGf%4#`1j}*d;5FJY$Nb|3q|?tF19e+XC{J0Wrl~%CGi(^B18`%;Iq3tPL>-!^C`~ z04SvT8~sJL`phSIKobOYSWPr3wjGGFxnnrlzL0CDvMIl6i{a>P?yJBh%)kFd|^FEtVYid|7RjKQ!! zj2#ik(j?`SuoG!dB(eN7A56THNcZ3hU5yflhL{+p996$=4(O_dvv zc?5eDEA@JB?SfNQDRnMz4D1Q1Wk!xT-0p@mmN#}Sfe}J~J|ar>2@O#DLbenNlQh3G zgN@7x61ASXgF3#wkvFx0KjAxfWqa$A;DzB%qjN0d+#RhFh~uT)9o^2+TkxS?^zI{) zyQV3&5-E+)6Mj;+g8iO8iYB8fcC}YaKs*zkBg4H}FGm|$M*!Hlc)?16>)%tGSJOad z!xxrU7TaQY+Rzy{VE38TO7xjA$vyO&)k4cZL?&^jv!K!?U%BLB@@-tZTrzsJ1G}*{ z#*Vl)ez7{@{T+$2+#ik0Cgr+jQ?h}@N)EZuJAN})B7v8<$$`%Y3_Z0`+Pj^I}hW;u>wk+_O631 zXV=kF8!6DiW6N>ZG`+RqF7#zn3sY^j2A>pfBtoEMrpqe}`hpjRq78r33#nwes9MzI zI4p{7!&<|wPsXd%I6Vz}aMI%1(71A~suCi{18_OtAzN^nl`9)+#~iSw;B-xrr2&hwSxjSQDs zqOK||Li!&($kDfu*2?|~cin+9UqiNv1SFiBBRaYea@7ugg2lb=Jzj2had!uon>74p zPkAdbJ3t0lI^3*_U3F%nmilAax;PQiBBTjdoN)h!X*?Z$%BlMWKgNIwOIRwGP%5MP zX{#p2U%Pf`L8zFCH#51bQo5u1{HM0|@iFYP-ip!_silApP9w*M_*9xd zw{~ksy4h8rhuXEf^^2#<+0LTdmFSc7eOScE*|Rjq;JCx=VS6BEv#;Zl&4Ub+i|=j= z^(uo+EN})+9X;h2Scc})%BEh{F`RthI6a`jr?4W+;5;yJY)Kb6+szzBNj_%wBBmm% z5tAS#5KdEfCw*pt_c}m(Ry%IDH?I;pJ6sT1~EkDA43OttdDb? zVSb!)2)S{QA4Hd#1p#|%QzLl7d?4&A-mbkVV?WD?)Dtq%v@s#EDWiOKyej{%pJ-F; zw@tOcH@SadV*}UY@Nd4!rPZ2ta_1*O{Du~sFsr;JeU+g@U1h{ml;aqPvn~#||Am>$ zV0^_Rl?)%CwKXJw-lGKN2hzXYbxIQH{++#+i>H%OVLJ8k&=|YP-5Ku$b_RB^VYpcB@kHk&zwmW zxlTQYUM@7*RjIy$nMYVw6+v(*@;x_SdW;z)emxDceixRGRD?{GK9txYV`73=`Px4hw^4p>Zma>Lh9%HRtf5ZL8wdM+)PWHw(U`Itv~`RAwQJy!gW_ z&jJMF=*PDkZCLsAoJ)^OOG6p3mRE1}Eyh;XB8wkU3FtZ)Jcf$vfVa@+rFdj)%XcAK zenU}-&Z`-P%#;rq!P)5t*4EA!>M5(=*kUTNvplPzj2Yg$iuMk%Esy9gD$Hj*6^7Z* zj*YDUpRoWM5(|FcT+&cx34Iz-1CmAhEsBWEhufjO=(u*MTqt=94Ps+0i?M!Bl`bpH z?TJ(xQ>EICOZ}rgBzLM=-v#E@Hevq&KJ_AfPElIJymqX$)}YJdpdx9%eLm2=45*+* zfR|H1V2Gq1Ot)?FH(5J!0$#qJZt+9jJ1q8OLf&6yFZH}~l+F%V7YbpuS3y|jPh@xhAJjH$9#U631Li*62k&RSpe%K)EhM6=Eh`9@J`oqa+o-RWei^5KsYFXH}SFwMJEEBql zkKWj#&~5mo_XpIv)Ci&%C;E=&=GhtYYyo=^qEO{i0PU2wWsF@g0ix+XvjkEh67y*< zH*mv%!VZrz02_5Y0$7}I$dCT}hGCYN25uM@r^)jG3V4a6k|FyB_tY#5fb_kkRt<~h zuKcS4bSAW#(7*F}_WwJd3)f0HjTOk$&B?30AqaFCsa{dU`bb4m#}=~G*$^r~Cgx41Lf2d~L56F=$X^-d4}Wm8S<*{orSiyCD@f`F&3ZI~digzQ9lIPcsaxty zN2=8VK%I;d9uurxBdg)xpZL0#leSJ!e)7gARn;5iNCl}$4gb-}E{S_gbRj;8*&cg@ zL~mLg6y+;4M-x6~i81{}063%~Tlw>YMjJyFH8J;8-Zb{}hh{nOgnCwKr;IhdBLC}a zbl6CcZIDgitjnb<&V1&A%2?uwswb|(@s}qifsZodtb9p*rc)HsvjvBATt{Vx>LHos zmwufqS+uPqc3HC9z+_e%PNGG~G=46cJJNbdt8p>fM;^mGOJ1sf5CXwe@F~{`s9j!@ z$Te~6Gc1LmW)|_eLz7lN0GQ>OF<;c|_#(drnD-5GVB_Mdx`y8Vwu(c4h%jr4E~#7i zoj>va$MbkR&fge9IeUP3eobyPS;b)Ut)rma`h|M<^z*{@<-mEVsbh<+I|ZIUpE2b3 zh_1l#Gak6p-0dr(X~^wJu8-g_pu?Z<`kkxTDdW3)`%$c#!j_j`m}`#mBmA^lNNWz< zEWjOPHrO3wHq`wk5Al+q#I3r0{kU5B*ZpPHb$)9o9+WEldI!d)hi%!(U8}M~=~5E1 z2rH4Mbu*{fweAK&u**W`nvMEI1LrO2;l?NwGf3Anf1H9*+I4a#?Oi8tM|?}f*eSi8ZBn9FC9eUcpb$*d>Z@q8OL%?{Q}7ViQ7>E;Ntq9|?E694mfW!=JS-)I~ai=*a80h#{^%P%O{P}&_turbg4P9NZR zr?s@H6dq7nS)Dc2*krMblWa2M-j;|p!q4&yY=3=Su`=6W^5$;+THF=7;H|c&*r*r* zfjC1FU0I5jr@*ijd8LQd{1?rS&+jq}{!eHg_I@YTh57xm%GijJ6kjGv;H1}S)|;a6 zF}Tun%EXB{=7*vpzk^hI1N%YF(D^30S^{rM$?0b$pM{8o-V>UlMa!w+$#!dD-{qeH+ zDR;tDDz@>gD4@OQ;iuBArMtI}O zYp+0Rf0FUr7mi>6Qu%hvt?NUVttmK}TE2T#GmT(3e&dMR>uScvCcbE~gu2Sxn;{>W zbYaAOxBH<=_gWJsly>YuP<{$UOndG)=HamB#k;34`K@+9!~9F`8N^g*a0-3J`2Mn6 z&mOvOO4a0M1lLzvyW*&hiD8ovhvY_BCUhOv%ml?L+tAZ2dhHCEJ-Z~NF+N-S;TE25 zhnf=#YP~_P73|?!OqLrA3`EDfq^zRo@%cA;t+%SzG(BFnjojk*)SAlZ=8HLrYKIcB zBqSycI5A(F-tTicY_1-wvMTT~`+0W`&c6Yj$0^y1Tz1p73?l|9s_}2qd^=J3yUdahTgqveYhASz)<-&2ctbPRWY+ZkHB^~}CW+$43%EFK0rch((RNH`yxVD# z%(F>)i;l=<%dgrNbM%%fhNp2ADf?{38mAjE=4X9~4M&>~RV$bb6s9*&T2iTv8P$zL zI1$6J=RhG7GHlErz>Fw`G>Qo3=UOwhs;L%`VOVw1axrzhRk|l z_t-hSARAK@G_%0)t%}C$O}9hw&s`Kj&(H5s83rtkRpBQCD68~dLPCG^j#g!a3o|Du zOdE={NhK-ewWqznQ#%Gs3GJ3D3vjCoPDNIu%Li(Cb-uCcii{2?ZL)It3t(EKQBtKT zbk%f2%iZw&meT&Fzi>~UOmTF`MF|{Xn{ksBrM*(SXzeYp`YWAfI~MRhQ$wTm*na-T z85QLL5^W5$;sTbidZ>!a?miMIv0eHn4L<@XGgOqv-=zkx1i?8>r3YDo(lWM_hhn44 zhC2c9&*$+p`Qy_8ICqqwd-3!0?VxXGC+dF~J_^nBA%w)qCgxvmCOt%qgk+L|&DZ{Y zIgzrf`=QA>Pxk#Yw&<-Ks~E6}*vwT;flft>OmRkU;@peqA#t>2j~u$yg(t9OnM?Ak zq;i@B?YdiAPg($1LXVkoNmszFy3+wFsP8LV^1VGNL{#stJu6Mq1e+Br1N8e51?Lw|$w2r?)-F=2>xhT1%s?DrE#e(o^yO(Ihamy9n zmwL+TH<^Ib8o!DmXsggk6Sn@fqOw9A*M0IHjRJl6W=~WeFguwFWaZ*|ov~Iq8g7DX z>6|x>Rt5(7j2}Oj+hn;pXzzM=xK^Abl39v3r7Jq;a;DmVtuS@uRMOUumV=E|Kzwda}-F8YoSj!{WYJ zH@}5^#35nJNoTGlxoLjfRWHU3+ZOWSjtvp{|B7a}7mvcXmfKuh4np z>Kzfd<;V2y`8nc=zp{E`*W6C0ztowkxmJNTyV$HpUA8BAeI95{@1ac0cnDt+j&F}lNaOvRxxqg}2)ARiPrhT(w>HqH6 zmc`&b|B!ygnOt0>2uC-r!`X@%g-C4-JWVxkbLUx3B;ZGaqRzy|CTK5aF~=n~5`(^; zWNP|{00L_^$Tj}GXgnkuMXOFn2C1Xf&~e#wB`Fwnyit7ut6C|Jrb^H=*Ou~8+w?`a z4+$(=M8z~#1!O}JZcrtL2xS|d>hov)>`q@Mu|ofM!d$t@IU6X4swurrR^ zHV8-v|u5E9|Gb&Xn;@lp&UfR&0^KGqW7!2rw+li2l z{dq7h#xmqAcTJ$mH-a(4GISdye~o|~l2Aajcvn*Jrwkn(2Ws9w+4>ZXek`0-I;{u~ z*;Uzg!|g+MVUO^=qU86_1ll5UbxAunEW%IhK0r8QQD@BELXNKo^P?IoS!EV_0K`)% zTvcwab48=K?p}@){Q)&257R>2t=Aq;XSuXm<+tTX@Ca||Te<8FNk6(9)%e`VH?{pz zKgl;W(uiYGiy*6{Ge?vdCnj+`Ol!3wWh{(%330roSw9E%3PVT8>!*w@)Q1#{Y8obt zz!L#EVdAW{?)dral?ByM*jxuGMkxTMv|*4@hmCfg939~6jIW`#w|mTxwv=Mi;SZ75>?;z?4?mWL?NTgRstjIT8*i$;dbFP8(yvuqgdU&nsp`j6Vykx zVA>O7DWVWu+{thV;|afR*5GHHeh&moo}ADf4>Su?bP3Z}x#3%89Hz83n07i+xgk?# z4B<&J81lIhF&7q;*aV^^=9t4CRk}~saH@+ybtVn~dNQ7lNBB!?GR$S|g{%#RsLaHW;Q zT>R82U`c7ENlWQ|g)yr3IfoxJTQAFoCkcGZ7XRY2>-=5<8d4o-%=zWq%bSF19)@O* zD>)30@x{L|T0yEoj@9X@?IUewP1YFl55MNN`Ht76g;GZfA}P;$`Hm`g_t#{52RVdzEiLlIYaY5vhxQB|*`zD_LTg@3 z$oX}p-rK-RCh#Pp&J$Zu#grlrDBGpJUD28~T3VsV-$%bo4r|A`GDQMfQtEJ8TnsnB zI!yGIes7gzh%;PGzA`_`TAzzc-@PtGZ>1!wYuGuqpNfCMXrLhUAt8a=3tN; z+SDgpZtBIWzt|zoIIjLbEPVrzCEw5X7(2FY+xCua`;Kkfwy|T|){bq@j&19k-~W4W zRX;bKPHvy-)2TW)$w`)?*e;PON7jMy8(pg``q4~e&nrl*wDj3DpHvICB~sFct?X8?NeX}0k3<9jss@e7!FRIPNG zd0eXc0o9uSsM`Oi`v0f~Ho95tIpvk*Cc_uUcQ#wu4e?^_1>v);%w_$@<)-QN))GE( zaGeF=^If&HMK4AW%YW=umCyFPmOeTh_p%iS9j7*0lJiZ1yHCqgY&QF6?R=H6<<&ga ze84WQIH`+XTKMfpH5PH29xHK-U0*d8e9RFRB78LXy~MQ^2(=Sd>qko-vM?=^&#Gt5 z@{cR$Y&NSsyhfLL3;2zJKN3~@R_jT-yFgJwJBEy3TL^0Y8XUK()qlf`KiLvRPd{s4 zn-N@y0CB3Hjvt44orm3UK0DHoUy$Gk!X5sNm+ifs@SS|fZGh>+?XZ)})C?h4GH$O% zxaR?X)(5zFC|U8Hs6j6jJOpjHUry;d+B1C*kFHk*?gZf6JcUB=aaK`Kgi`?HzY-5c z({U$Mc!YDH*@Sa{xP%MMT|z|}Hq85*h13}rnUHM~kp-X%Q%q6<_!9yCd8EG{Qbd8j z5-B&$k!>zE;y@CQvz{=i9%Q|!#~a|>yqn6U`6Cypm1rPSe^>kZEm}_9(?b;{sKDV3 zK$&mlUIAF@9YM|ANSIus{NyrOhe#o+T=+0SmpimzHKKs6me4_$^#FZZ857qaFq<~t zRLUlLV1?g(HUy%;=l8nGW`1rzIhKx}IEtd22>b&&Z*nNhOI$+;Y1ePYq@J#z^UPt# z1smN=44aFI*gV6$A10gc??%&~zjAO~DtQEcvAlxY8isSh96bg>#o?@VGu7(!Xm1=5 z(C?hWty^B)kdTd19J zmU|rM5V-Dm&CCNCYp!rr1^=R7oyw zSpZaCVg)w^g_Q=716g@OY@=(g17Vgd5ij%`cb_NVWzei0*znkP890$J-qblu>vXHP zE&wBwoYFfCvQv30`aN&wD?IOSh_YGbZ{d&0L|YOMX5EsHO5YUgX>OA`9u(`L;esOMuMP`z;OR}8v64VV9UHWgRu${4MUm>fRSaMR-D}) zqZ;V$cFf&%fU|~QaR8AIavzZ1rXW0+PJ|}|#$YP_2y+1ENBa^i#c>37xM|`JVzo_l zXtkQc8@{e%F(k8U&R?scFJ1`Yid8xbTzkVok0UKO!4#3QX+{|_~=3v#7ZB|4Cbi1 zJ#-nC#Sn%OD;}R{?xo|1RG5M*I>juPI5mpYrp|w?HB`Yx;H`~FrLFS!FRiT#m@2dB zsrLiSt45kJ#T9HU%}#9;?(`3EJPE#1dr1s$|4L?R_jxGd#}#{ER_x|7e>ir)wD4lxP-+n6HxbfCnuVboJ%%(0?g)qW3YH@K~3$I4+3Jg|6 z-^7GIV`(8vQZ1%o8asLr=Xed^n1p0x5Eqt=0@Kq8OV)qBFtP|o+!b@4mnCZY-XXOk z7Tb(65XtG$+=uPb5T9yvptCnq#Gq3Ds#fuL$s&rpFmEN#p_bnK zEkn^V_rQU9-Sf;@3o5D-Pi#PvUq_EnX*S_Euj@pkwNc|viVC#;kb`q3nKXJ|M=|{M z`;hkPv)Q>k*<5kGzVo}?NL>y5^67S9*wHW(Rui7|2@t0xlcyZ&RB;5})Ucb$3iYj? zD08^~6m6VbIco;ree9c7#Z`hg(RAaEZ`C1mL;yakb2%I;4>oWgo|_7G+TW=O!(>;r z9Ms4TtGGI6>lpqA5xLQDrY15E)*AWU>=ZRDr#zDZGdml_a)>0k-aXDT*p2CVjTxB8 z1#Bj0d~Hb%BvD6LQ%yyMJJtSDY|E28JESCS-@~D7ogMipxhkw2SlKT-Wy4L8a}}F>4rKvI}cxT{u$CIoRy6m5@4F@_Jz0 z9&cDqfRZh;t_>P7ma}F6RQ$U>f`;#6_z~5v5#xF-2qU5P0Vk2VAW~MK?R&t#?dtH; zP|UaOL+k|Z=C%AHn07#-eZFO9^;C^vsp)s~-wrmU^yU+SftS8-X4q(-x73F6<2TlV zN?L`PC0e~GCAIkhWB{A(Dc^#q45BO+vMCxn9E~XO2P)Y1S=7$J%W~?wT-*f3;lq@E zrDf_!EO(IY`+gYrKL{R;fo$!&RdjK@zR&6h#hIa#%0`jwZoPMM4)rV$`Sfj5@5`Jk*yI*udJN|`#!_w0%0I)U%-V)C0=ZUn}XF}>Nd1daQLut=)XqGmTtizG@pmuLcYE?%MR zHF?jwUh_(0nvs;>$$3@jmY5p`rGcrG(X1`U7aezZqPR{`J28`*#5(y5 zf5nxc+G~9?j5DD*M#m-;S!X7!+bxq7&&!?*a+OVgm*&G{TSOKkVT+KmJVePampuT# zvF;~od?QwR?X;6j!pt(Rve{J`mUZbbF+P-`WwQ;^AYpg-qBaY&G~jwboHjn%s>vOBB9M$? z+-sg|8|2EPVfeQ@CS9qWZ8aGD~-$ij0Qh();Jp62cVw^ZELC zTshhi$ncBn`5^gtKbxuTdE1|x^Xq!s&+C1=U(xq{Jyh5K{yUc@p^xYDNpJ#yF@A0l zrl$ScKcO6^Js#6NcT(CLEBcd1vhdN{oA}iJ9FP36u<9;}mI#$4kAhn=Zhb={L5|a_ zjT`g*@%XY-!1lnw$08-_$$1=2U2}WKxff_(F7evtddH6wBc7L{XH|$u`G?jgM($Vn%SU25Y@r?N5qQf9i@K!{2_@yKT@gu8?b(&_ek22?^Hr!G zZj8QUz5i?CY7dy(ZP)6!}WPkP=`4q>>y zP<>eY8bjnXPvo0wGu~eP&qKWCZ4(bQsO+UuTg1m`7}ff*==`UUQ>8`^u~4zg<{4|< z@|E3f=+uk{mSEbd{4~XwfDjWW3SOF{$+-HGjh0#~5X+Ld@f4y5hOsqpwZ;`d6);bW z{D%Hi$%n?fZHy`av$1H6Dq)e-E#zRa!E`kxZ>lZrkzwW3@pv?F*LjVtW}qV8IFIYT z!Q&UFu4{Dfw*ElI8sX~#hu3#6lY84&?RSVPgXDmwFiYaq)weZ)9@aSDEd0sMUhGzb zmiT=yB-~Nm04EX{VxSvor5ks(TbhwrCi~ennt-gK10u5PC+hpbGG^0>4&jLEIYefu z<$YV3ZwkyWsspL4sSc*4t`SI^Y^&tIKPx+`l&zJXm+r>Doa(k)>Cg@y9`w5 z5CUZ8HiR9B5R@AT*9wXjkkH4XsT0eD<3+L;kZ6N57CY}BoZOI2L2-J?v4D#)>-aK# zHNYLk8r7}>qgL8@yF2Kw!7h@~?xhY*H@*M>CH8Hj}=D-Fa zJ1>w2f-Wp4pQIJAa+XlJn`{#bjhaI|--GV?PGSj9lt9}#j2 zltB+@?hZF;Gk2l(8D1?hXi2r-Vnb~pRVg(*cy}}kT~jZYd${M@>>*`(k5G-yDkSJ7 z%~?27wg(!lONc-1MR8X=?a>NI746B!zmU)u22P@~qdeeJ>5(fX!%P~ytel6G&V#`p z@9hguKn4X_GE0IBBtuJ8Wn+4PRB_ys&cIxGLh4$a$^KMCn8gDmljporqGr2~HLwNt zkX+gt4gE(<)f#3fGS`Vvq7Ao9(`NZh$=}8hXC3im&k{|C?OM3=iwR;BOey;lHRku! zAP`3Zh1M|xXYyv#*f3_HJV~N!1vqaE&Cybc)(y2Qz>2i(yD*spw+6lmzKDY=i|!HV zcynM2lbUDYh#g+%35mC-O_C0%M;!JSC4p=R$|ab3TxXh5bY(MbD`*~SSrTl&Cj*ZA zvOn<7k^XI-r;-RU$1WsH=yxxetwEI)u2*tEu46QTAxKRK_Z{?dCmPG6?wKUKhkC#nVdk-o=@d9Ub_m61*_!)~P zC2Vl_oN|Rb?yw|b%0|=e$y558=FoS1FtOyv9L`0T$?r$n6p^7B-pEuajK@V>CCZPw zhLvXR(4VJ&+w~o|9YXSFF<1r@+4p@Fn7~mQ)MQ4PRBU<^+S}{*+^Zsmw zqN^`-oX5gouLMd^@n#rDJ@D7nH3*5gDjJWE!8*8e{JOP>gI-2c;uRivC}6+2v* zI^GvLno{c`9vPUGzJqw|mk)X4u~4eDxmXxXd*saO_bKj*b%9zRg>~io`k} zKm`IrICEig?!Y2AG2gV2-ev6Hwja#K7Z29}3|I?PV7Kc=6g|oYZq{YP|MK%99&>T? za_rrl#>$!AwC!-jeN>+9Deqo>AmT^(OTXU6>zlHlAD-^T*cZX1r(#CrY&&=li>sLA zq`CI=AStGDDb*%Vl6xID$7B>D0je9sa`m?b?-W{6?`et29T4q%+RH_G^0bPZHct>4 z{wlO&cG}Wis@&u_4y)0c(M~m)Rqa$h9ar6RK)nrhg0gD8HCXeI68*jdP3^SAc-n_5 z4X`0;-M>TU|^30VjtS$r5Vcp{U(v54*kHcRyX)KzX|Whu`EsX^lVJ z%i%8T;yu7l9ZZMvJ`OU?s0vD%wT+K=5=xG^fj|RG4-UmWG>{llFdy{95oD3s*c}S% zAY0xLmy3Kv&*YwLiU%{g8WU==5?p3t#neCmL$T+zBTAc+HgL68%Ne~8HKSnW37})< zp=9PsSc3*6tleI^jb89d<~sgK!Y;yp@0^3LIUDq6p*b&8N7e84pO223Xm)@bXUD-J zX3Row(8S)o zgl5wgi^}=iNk17Oj2BQ^R^ndb^p7q$6iDNlz`RImLIK4Gy>oh9yipD+wWjahn@*}@ zy-qlB=23=9IX3>p0tM33>R)q)3{xY&%)42LBNbYP+gb@DDZ#=wfRGY+$#67~QFWJA zo46sL{Z0!3rbbO=48JY7{;Kl1E39QHcD|S|Z(of`?iBxhHu&g$Pl!{JfW8x!g)$io z;Si)cB^bu4Qd3_dD5r?1tBM5jkCmnPk&hu@spN0l5!cL%v*s3c(roIqlX>3YZ}e6* zh}p11Z&SNJ8H3G~ug@)%uWn70uM%SdQeT&JQeTR9t@2;uq~CxefL+4csm0F|w)r{m zh?#jnA!8@yOk8E6Ca0wtoq}u32U8yG?f|;4Yk&5y14ivN9-Oo8o&Wf!WtZaJ(u-^6 z2m&QWW!R->iZgbQSUo4W+p^w5p6p~Jq3Gq`Vw{Y~>Q-i#{#eYgJnzWFkUJKDNw35w zyvIEQEA@!ABYg9x9M`Y2+4M_riJ21-i#T}E=hU#iD*EPSn?$|El|@OmXm?oUzZOzT zze(-jaV1bJS)gjN9brOZV}k~D&Xn;?sw*;=EYrQ&PKwjuZaQCAg%}8DTGAd*H0|>r z{>_tp%L@%5--06*)1E9?Xb6B)%eJ-k&=~4muML68)f4}?t`KOBr*_zZX!hTp^QXfs zG1%*?3*eMYAn!M)%A0VUtRSAT03WP2`J8t^YcE_%jHDg)YLf7yTy(fQYMpuKzIBxd_`+t`n= zi>b<|&Uag!P)ObNZ0E9HzI$%`=E3=RIkHq%;X&F-I<&WP-d*$3#~F6$tvl7CeT2=` zsT>BdqE30#EpN~X{n|ld46^@?D5`Aan3)k1%=7?(-h%2r@~WHA88n-)=-aKBdeiQ- z%LX6xrn8R1!yJKZ1MkB}sz9&a(MEo>O-MTjXWs>T*&p>jrCO<`-JOnHguW~o!ae9} z>4=u7tMChZ@vy!U#9#&B{>-TCymO8<0`-J*kd>T<7Eg!ofESFl%$F>EpQd<}SzWEF zb6=c3cy-deAf;VyJTP1>Syk%Jb*Oy>LNxxTib>LjVGx3Mnp8mJlIAvo=5fnE*`c`x zDTH`F5$tZZFHkd-b;7)@knbVn7;3OJf3!+}jzw)wBOkiI0ufBM{!Ha>2fP0g_V16} zW!bQC3gNg)P%+rGNFpqCB*~VRv_R4o_=mxp11&csRS5ns(K63nQ{DZZ#dU15I)+6?)UQ{=OTHV+_Mre(*^l0)Ef{r$1?sm0#mI14xpPmTRQC~E?o z0_^7InGS{lrzh5P^M=|@&Z5$Pum8>~MiRu4WvPV-ou^y)GyU@P6l$h6 zu%$0i8{T3&_47|&r%v)Lcd34cjH>KKDvoN%;O#*`5n$(hojkT{I^vdJ>uHQ%yH%|6o?R#v+8iXGS9#oNm9Nc0N<0_8u| z&FR=bb|)93Rv)G2ROBKA(NM~#6w4a?#ZO`%1kN-vNvzDuVC+`B7BN{yO3n^a70dF# zYtMX+X>#~)&C??i3i_a5R3@Nf>p;k<;}?pFcwUv#6pp@3OOugwQ|XvkczhKjPL2-;Be7ql!qWHFOr*1` zT}=i|aiz8CSfF#xvLIQ1etzEG>kB^3>Tv>yPcEy+c8iQ2=%nr*cL>EO$Nfb~rM~h& zhMMvcAsw1$dd6AEagGOZx;Zl*U((tMm#jFb+(Xg<686z0--9)u7~*77yPJU9%|z0>vp~1Nv+=8=fJNQJ zAIJ{gsAE^cP8NTS5an}7g|mKOx*)~P1+a#Xiu%nypw#e_T(CJC5`#mkKvm)n7zwwa zAf=Xk&23l4@dfZgYc{Au3s8j?Y;WY|>l(33FTbeu=P&f;YsGi`=4(Sb%N=#?#U6=E zuf?8k_@?VUU~G|@C1pr6UR8yWpWLlnGR%GT3Jpv|cczCje!sq9CrE8x6`H8M?>b)5<-P1ZwG`gLW&u(v)Q-O+pGYmZS+Z9MwDkB#X~ z`ZdfC8}5f~X~DFkKKcCU|JT?wsp0eN{^@VgXZ2+<>1AAstmlRusgq8X^&JS4NtN~d zlHRYx=oDFB{W4}X_J|Z&Bhw_UsQ=6oU^O=nOsB<7{d27U7$d9)lNLAk53!0)hc`EW z`(E!nG)`XM5MlFUZPNc6;m>&`of^A^_0NzetY*~LiqHk_&Z+H~z0c}LB&!^dt+I!H ziVFRYY^-8yNjIf4(~QzwHcD+40{u&CRd78-YsEC7HQBJd{ljY*{-3b+yHCki9Ge{W zMlCCcewD)=NS#iVL&LfrNMRJ49QS%H>el}k{JBB%weHVc{dwa14}50(ex~nD?*AsP zwYAKLLHmEBU~Nq+(FkXgGm1&_{P~R*_5GrKiEN~TZK&KE8?Ks5GV;be% zsnj7X&%4kZT=9#Zwt^kL@&dyT5r{Xys#X*jM3MUKL-R=<1`m1s%Yo55XswC?^&7+~ zTQoPlTQYa;VL4vL8QFh1ch7z$XTsaRzz-qH9GA_)>P&f)wDJ9{CNSI z1gp`cWFTgM*vBEF~X|hY?lYE>Le0E*e=Y>s>TPRP)0!poIg<@YB+|gL zNL?r57w_mY#V~?~>)Kf)`ogV076=vNuw9o=WG>y{WL36HZy1saoMg$%Hj( z9^+uAHxeYRDt2|AJ;H{GYTbRsZ(XJMWqmgvy=4g8NR3xTMQL2>;-Cp*61Fo4Xhs4 zx|vsGSiYqOE?qk0CpO$BPgC!scU>fcT#uSHEmp=z;T2hcp0bTYOGSSNL2?wxkQdpc z;5!xc8opg+0`;5$jDz`4Dg!Xm&0g5=(PsTgw0@jqcrkIt65QlN-qlu41h&e5}6>IW`tg1*Ch)dhas7Px_GMalg2?je?_#8yOD8)`Hf zo#03~5rGt6HbDqvg%|p%v#IP=Lkt+Y1Moj{gc-?3ir`| z4W=CZo4RFKhLPlVTwqvjilo;X;Ym9r2p8M3S zO4@TAZ0_5%?Flg_7YpbRy2M;gXxy?$`kw+mCk4ixD#kieTqFP?2a@9Fr>jgf9QW`d z#LR595q55N+EJ@gG@Jmn3kdNZrtGQT@zMLq;aMdlUxq@#`Q@sKoVx(eNmR5<%zNPg z@}cCAL$29p`RN*#zl*)54@zRWPKH%@tMxc_EVY4m40b zA{qe@T666Mi(Lx4Ae%+H7Tvzho&TnC4YV2{>%nMcB;>DA*wthdS$RIs))dT$KeGBZ zj@N_5l0r)j2k^||jG5@7Pv;A%2a+{};p1IXxNytv5NB?aFQlCBzGAk$g6!%1Oj2(C z^VPUo?D-7^g812D2~3&z)Iaav9{#=Ge=dEue;V_N)s#|F8D)Ns%NefWcZQ+nT&M*0 zLybmnSy)z-{%asaI)k2RRb`(CO{H6YFU*C2Bf+N4(h9qog2*?C;f4jC3+Aqg3)E++@$LI@h#aD~c4F>@X9r>|Mk;X_^O9FF1 z>G}j_&RM;r))r(^S14^bklyOm05Gyiq*x}>n(bcLhMuFitw{8JYZH4D6Zh@A0rQwM zZZhMSs-Xu30$ek8vBsX_VGTYuyLIx-D{iB6@FgDR>w@!3I~#9IonK@F68;#LMTlb@ z9b4bPqTR(>-=gvHw}Pm{*==CPu12F0J4bv>8;}kJB6cLV+)ORQx5Mq;#Rs=he7KM$ zol(5wLLPJwVv9TFle@XDH#NtkU2zI@w^)NoyLKnc_9cG!a*5wKy1>7V9cHU+#s-8Y zR~Sif2P(!U2GU8dZr$7Ew)hJCYKt%9Qu>Pv#z(N^IIziqf{ zI@!_7?8MNbZQ6MVWGZ?QpeW%G9+)bko;kmBX&J_fcDn0(0E|_u`DhBT1t=?qy#(uZ z_C>o2MdDfUtso)b)UXeD*A(1`2OZS{=QDQ5FU&*SP3!m@T0gMn*~GtzXf=;*-vZFp zvXQs3vDDjuY2>eN3BYLF;BH>`tX9QuXQP?ST1RYLwmfHds&2Vi{ZrTWM@UE%fyx`W z)tvJddLWc6gqd=RcsRSXHU~ntkV{b`s@m8`7ygW(-{k{n>xDt~E-cUW`|0yk;OqHn zhGS{R-yR1>^f18RVmzDFctqI|p2a^d^`1`-S&z}wPeB9Ck#}DyNB1R;OI}?794s83 zq?gQcW0!u7quX4ZnZya!uV<;#F+kPLXs1Cjf*-m9wFsu(;?L_r-<;|oIK3WtYA|lN zEV*q1j{k1qKX$8FqtkSwp;$qv~SZ-@|0mzVD=?!!bv$ ztJ3YYvZ&eeThYwirR?ikBI&=;g!7&!t@cS7;H@$C{|nzf5J>WqIMJ=d{1>Yk3n1w0 zIugjhuu|6W+b9FAqGHwNHo-!!=76ntSI9vyD5T+O_EKV5=y`&~FsTQhiR4NsNLY;3 zXC&s*2CT_m@k7u{o(7K3vVx}534moAm=IfnOaQ8SSqMiu0#qG=LAn%vmZ67nMBpY( zyLnE~QV@BkU7@S`A<(j1YLQS+*q_ha0^c_ifbXpC?z~uWzt2|$uW#Hzg75b4sV|1B zL4v2}=O^E{ug}&Mj8cJxPkmpb;g4zWH`8kh$2TVfiud;&ZzR@kY^^^sF>RPwv%TU{ zs6>5^K?9Osw~1lmiey0tFsuziqJO= za9xkBV9sqsdaadoXBLIIB}kl#^7mfL3yWQmp#9h>tO$4|el{e1eSG5Y(~9g&>*p!k zE~N37*{WFxeYnRlc8Dz}4Y+VFAofOo{!t!#Bp5Ck{leQt+D)3~4zf9oH(3>OgHwxM zKjEA}G!0cchu4y@XIB=Ao(!*ZK-4&)@@pvmThSwnEEy6~XwA(&*B@%p5GpwC9ni2$ z)ZMpxKUbwH%*u4H)NVRJi`GMuqc~La32@xxOy9Bu#?EO7A#z=l>9fQXn!GhoW73LJ2 z`0LNU@ISwriI_^e1fu=oO_;5if0%*N{G-CxCJRMX`!Y4)!`;EH{ zgd8z=9m1)R&8Sl0$K?KzypSv+sQBw$sp};0I8{6eiv&T)?aR!s?kgax=CyLcgasoHT($lo2 zpcz?%s)miFn{aetNO?fs--jWVs(TYnUXA@VYr`8fH4#NC!sWc$6GnikP-Z*|=jJ~> z78nld%+7UuHZYodd9Bv%E(;khH*1w-_lrs=j!0lOOYw4b_fB5LR%F!o#o_~3d{(3T z-udauRGflTlU#OfQw9cJS5`)&iFz>N6R9=j$Oao|R#PEVroc@Zp_g~bS6Do4Vb2s5 z_7ahu_E<@migvMux<|ha9!Yf0JcB&oo=cKSEt6va1w5tIin>A+Q6pS>mw%9UDqIjn zrV$?@h|ML9*hmiRq$HXE;-vSXlyq-kY2_!3l;>%eXG`G!p!(#xga^K5#@n7$W)Nv+ zhLpVnT$|Rvc6H%?ISp>S`{R6GK zhFk~xFNApvwC*hGtGZ>utD?%Qv}azTt`h=>s461@oGTkCyFA0;+$cNdOBO5)%FxVj z%@Hjp0KJEziV1Nt#ftQa?v#`o@QCgElWBNMr$}=m{CtpoDmW%Q6dd)+emfF9v7J#f z#g1s(BKt2VnfjQJUa;d$5kq87Ey?k3TSS76QK*@hQ`!n)c+ z!Wp7M$$qVHqxp;1GldBv=(;FAa%OH)t{&gZs+H#g;@o(D;1g_YYN`2?-^_!CuST<5 zeu#ukUs+cpKVTE!VjWA_NazCw@`)b0lKabTSqPy5u_z)AI5-8RlWeI39-s-!+#k)7 zxf}hJRa7?0)d-&;81j>E4rSdU(>}Sy-=mTTKFpQxoVPPNd2t)XWCL}|wku>F$FWfW z?#lfMcNP~q^FfxjG6p(`vK)kh8b+$9L~M(QtK%R!M11i9w?2{W2IEu%?!}Mr?1g{v z*aaQQWa$;4!PHAu2rGbWiMI`0gVS6ini+DEd*n@8AT6KZ;{ zCEYLsDG|Xar1k-)GM21dnjKf?yk1340}XW&Y%38O18fUYoIU0a z@OxFtL>@cykf<#pC+b2{+6z7s_d$bxH0588iC4Vlkn@TWYCS4es}^$=f@Iv>$IHK; zC*nS(W%v&sFuBAMLoXKtAa$sbaJ^jphtZ zc6QP5{s&!}kQ8JBE5KXqAh$#52tz?{R!GA5tGo|dhzlv%%2JUr@szMpNvg0A986 zc~ZmkZ4lb`e*bjeKnlWGA^gp*M{WGvaWqVjA?u@ zJL@}TBr-CkQ27`s@b~*9$_9&UaN(n1(vSk!eRN6qF^dRq8uY*VNd$Q#PpCy(c*5{K z?zEUkIwAMX^}Aa0z)`Fj?(5a~_s3jjZib^bN@<^%lYw`*-!qxm!Ilp#Z$g>Kqu4PP zC2k1Bg68f8wdu5p!SU0HkpQu~Ps>yg&bn-Ov=ED$h6e1ZlAc?w{^D zg*ch}VBa~(S7$qMiSOX~I^@kAed62M(huMn8{s+X2U4FK1+y~LK{Mk1GFIB!D&d*Q z;U%JZqp+L>$&@jScylZ4F!xz^&?TMzNg;z`P72NZ4Vf;H=P!%E5@EZ7i`^wpq+=8x z>yYm25O3>1DdkR@blma~fiO?T|2q!T>>utVsNGZ(*J9P{KTOH-vO-QT5$3M0v(`(v zfQQ%2n3?by7xhc57K4%>?~ITlN39?alLUay#pnfzW+ZH$GOsMl1teV#Aq+X!OV zOT{Bsliq>Hw6i!upHScXrG)bnk4NnMh=e;yR&m!V-;@L&dnFT><^)Bukn?E0lnfXB z{`4QD9M=lzY_UE@ykV{4aEt;R(kG`%4Fh=K@HQo&zjU|Xo*Ga7uP5$2S9HZxGT}e%=hC`pJ_Tb;Ic7iD>f)9YF8Hs zXI(E=Q=B+V%)@<%8p!~e0y9|SNn$qPP2axMd52Ujv%hnhuwMiLZ6nU z3A>!yh;Y~coNVG8r-rse*4T)1NAh&N+Ixb_ zm9;ZQ0QDh(ZU-Sn*EZ52@rwVow(wJJBbPF_yizF!oE0Falt!zrdo8Hkmz3ONy%fE{X(v9SJ?PPQkC z+$_45s~@=R?mYyV!ULJbP;$aipF&?prcwEB8phMZ`#hYe<>B0uDAzvn+3P#fcgX(> zUt_lOh<7f?l-OW_PKp7xLQhl5md`0$o2@*LK(V&!HB4g~IJXwUQ{rNv5HjTNLXN*g~vk&o>76mvB!H{oG&0r}p=0mFE3*}a+!m)j=_ zdGbxVNXUiCaDtOAr&3`?4t%>t$9@=PTY2-Z`uy1f6#Pad3=by|L~?}Q*4toK6eu4y zDtbHjY3r3r`$NuoQdAC|KShzRZOpG`815O>-y9~MB!0gFA+PA$X`tjzlQ{2qTDb)N zF~)sy>BleHN7PUZ%x5c4ZHaPxTSnP_%}oh(ymB<399?TW-^ba0P0@9{N^pEnKmF@! zEX?xzAMbxMb}pfFQ#&muM>|i>|KpneXkY$F*ZwHm{U{?WV~KHm>$wtkytX#&kp6@X zKC-m_Pk-nCWPWbY^9fP(NADL|c+odG2JfFS=F#@!d35ijC)MP^&fMONY*Y&wZ!AKpjx!;P#iaR~*UQYO-IJdK z!B<^xZ|n0MK0zc7Ft)Cs?y8r; zlXsrK&xA{RTnsXB`$Kn7x!zd4IhSdezL>=l1+CL>l*1w)S)4=5c+QuY+J~YmOz1+1 zf7gm|gQSLzRV=9F?}^&5zw3zCRwuv{m^Cuaj<;KTQNy{zj^vI3)Jr7XGc1&J@M$Io z>Aw*<{~pq!48|tC2gfhU9jJS#i!9Q85<8q&bX2*TaXFu&LJ$&q5$N#)gP#YMfKOKt z{h?p4$J<2HbBj&0s-iP(!VdD>RlU8*7wak>w>jo~4^C?!DZHk`$Y;_!#dO$1eOINg zrmt?D=#~{M=V4D9eiGB|-bGj8HaxMh)$c=79FHWX(GH$?^b%9?$q)}i5GsB93%zm^ zGzpsoEy*37G*Zx6z+h~uo{Yvhk(x5jk=^VnepTKu{O*uSBvc|6TB#JBW2lLR;A4=?*Y8e+EalTxb2I8drA;(G)bB(-RRBAzKFQF42KW?vR+N?Oih*Hk$u`=ywHji zxiYulz&4`=U^D6Cec&goeTG9_=?2$=;k?0^g}&cEJ7X#iq0J*VTep`KE__w5YeILn z6vB_m8|=siW2wwMqzUB|^b1_AUD)T!HZRZ&RA)b`;ICwYM&@4yz;$<{8C5;@L86tH z((o&kl{L0gin5{|b5oPwW&?nFh@a1h-dOXMFaqfT;w1+v#hT7OI`>9`s6JRq% zCDxfP^G&O(5kfJn!^uK58Th6#mm1i{O-KJ6X&;_A|1|ddwPu^M&$DT#8ZX$&LhBfP zB_zlbJpM+OW#*-GpjEWmCNXTeQp*>7F)t{O5Wg>G>-Bu>JWN%8x4)?u?fJe^O4Gjr zE#&0td2xS?tBm_=x1E>cwn5WQTr*i^E`eODx~y#Z`&6S(yhOra)MV zg)>2s#AcKNA7BI}TgWl%>rxQsL`40))?L~F(j?g-8${o{)>=vjv*iy&{`>{r#k$5u zf&B460?a?8+i^r(7xR6`OCQ|~D|2lpDVQ~zos=ytZeR$!A~?EEdM6U|E+@isNqlC( zm0t+99_CZtpSIZNS9nV_naL~DHerJuBSScKX#&Dm#BNsgP?6+!`67SDZ^yy0stM}j z+e)RCDONp%KGgal6@A&!+M>`8{)p_vEBbx8BQ-RCJmx7I|^e4(eA z79FpIctzlGOmfQ}V34IV@k;`aAq2R?x)_aBx5i4aIbmu0i}+Vn3`(T9yln3Ec|-r6 z)*X=jE$^XJ#eC@MmW)1$IPdv~5=76hlk@o<79D+2znQCe7EyThFo)MfmBucqu$gK; zt@%$ZwR$*L2fZqns09~f%IkD7#0|BkS5p-dI`Dx+Mp2&t)yMpE@S{(EoLajVmemc`Pj!>arN)9{%=-5(w&*Ad$jTFQw{W`&WAcHdJpY z4An1J+FrD-lOjKnwL!a?Z6JCYYe7(DBk0%rWhZw?3wPf-L;NKiw1fKCS+6cp&ZJFg zk1EOjlcuz{W9=F@F{h9KmFXnco5SZ1DSAZ{QI zhO;vP=%znDf=w+-bG?^V;L+|Y+TL;+Rv?Bi!iZIJtq;QqK>hgVsj(hpB3G?bp?xG; z%gH?7jo13EKGJ;h0iNk_H}t?}#mLbas;77aC}C~cclJNB+1iIH*JOAtPW?XS)iyDd zDFwsSyVrG5RtdScJ4^LRkXw8Hz@-4%-|`LDDJai!y#?lbEhrs=SUqGJE(;v}BA-%W zLK1f2A}j;|{`P|rxscC7XAA~Q*&sN`mE}mq8fcUKw5F26u<@E|$)v-#O#heeY&LL< z8d6DQ{+U*5*KT%xypCf96su3e&2tii!M1v4_oZRSZ$D~Mp!j}j-8vGm411s2r{c6Y zA^pIu7G$juAw$K++8)w#x@3_#Mnc%^LB}dH$lvjk!A~1r9_A+Xx}1%+3T_6{#?8>3 z*wz;|%r}|l(*io`c*;!LKYhjU-DDT^YLMK~95?EKpO2wzTY&AJ%R<*e)T;-A!Zg{C zYM)V{#xcw(TGDTj4q|(=a`6VUPqd3Ev&Jhf0vc&GNOnlQ+el7nI;&Z(AG67QNFJcz z8T2_vS@oOX2H10?y@?-G(1zDkw{;P~Z2TL0eEeK=2!H{Na7CmyJOiiMQqbcxjy(Gx zHLV-n)6||MlBIXE`0ojN2mFP!_rCRnIm~3$X5;`Q%G*l+&${OPs?vcsX4yxI?myap zsLnf=@??#gKLqc2R~DA-AW#OQZ5dKp4I_|=!;0_VCp~RD#i^Kxet`{S+E<3LmgTw; z(dGEKxc#oXa!9qqazib+0s* zJ!!?60UdhY&+us`AX5I6HropbHESk0uCHX3t6c5~dQ@-b&lE!y+Z8%ncq*(Ial46u z=9JF=Z1%A#+%5aZ-rQf-pJZ97Wr$jX<(q#6$5;GilV#*{O|lf92>I7VWl;H~!{Q>a z)Jli-~ zGeUU$>RyE|K9cn#>ELNFLo#xPJ|X@LJ=;ATw+uX`_L6sWRoa7FGG?1mMf~ zbWmMbssOoC7AJb7P;b9cO+=Ph~fcF+qP}n zwvE}}`_BKz#6(QQ%)uNy`&r1$eXv*TgOyj_*#=GX=c_G-a`@e;?i(lK=_Fmdk{{z@ z=BYsdU?m;-bRtBC44CqwU(5`>J9TZ~U(7T$YWP(*0cLPl>J-QyDM;1lI7IZUQ{b|Ji@IVkf`_b5|>BWSy}*g>N;%rr5NtjngfQ)R&;QYR4^8p1ZD>;8d4dwLSyB?h| zu(yd?V%V2utfS?p?k~zT5QRjpst86EtS$6=pOaCnYZ-_!K2erAuvUp9E&2&IHOS%2 z*|H~23XzAs7Tq|e3==>W*i+7Rj-pQam|g8t0;RZEd=lbb_UaZt8t_t>83K|HmZ29Q z6!{D@OY%pFV?DZ@&;c}}F!-0AqEE=fujqM0#N zx~oMZOxR~?;M!-W|1g()RWXaxV>dd!o_6p@Ee@xj1*bSWj5Dm~ijp?KuqQ`_a(Lqr zWtZ-HCmQc`%VURLM5K5~F>u>oW+PR{i*r-yTR`F+&6@iExtvr}(Q~n*SmF-dndN}0 zM)bl9C)3tWRe>)pvME zJ%1=hI^X*TuPd46)pU1uT=Dwa=KzX|I4Z3-C=M2$*@LLjo+EDI$la(_C3IZzSq^fd zU>ki&sPE7@XCfyTquehBFkvLs4%~ze*T&&Mr%JCpS+TnvuAG$jmOHknKE081;a)WHg53@@8_Mh?gu&7>Yl$2X*`ilmVCv~7EIatUQcNaUP$x|yfHC1>7}<3xO)RR`TqZc%$* zbKVsF!CqGFKJ3jN*cndqWDPBqZ@3H;`^17XoM#A>-l8=CFZcQIv9hE##x3Gz91o9G zv84GSzGO(exYRT)%lCS96q2G#@mjo7C=nhbUede<2%9lH<*~9%@djrWBG7Gn_bG4S z=b|O-yBIAqr~1MQa`PuoR2kuZMT>JEm&FXU`WS5avxt z7$Y~AM2DwGub+3nbTAmbM$HaYsZb~M`PX>i3@34SgoC{NljU-*dogSmRu30+`T(gU zG~rpG&U#}Hn(ElvVW7069rJC95>8h7=gTDJ4$=(Lf53?Hf zb)K*-P-#!hdR#)S>$;?iG@%rCN3abn7rZ0BU~BdKQNZP$$uCpob5v(yrW!=UbD0JV zPC}-RUPG332OtnkBYgx_RO_`E-*8FKffosOl%0x6GY;z~n%V&ZP}2?;DLS}SBJ|V; zHeAaAQ)?A}DSCrfR2xB`EIDPhvzo;Nt(lyPPlom03$TtQZ&Ddg3 zC6oxFhxoB{2L5i!r^0HGfYF&(g+cLno~a_Yq9qkPtwI6^RH&PDR8A*_zSL5&OZf5k z4%yX$X64XWV)EgY>$*B<&2y6rLF)slA?ZPhws{nX{1CU>W}JDer>!KKw+)DdBoVmA z@EEW5%>ND4L9it8jpqQ;tZ)tk`Bf_)RMwgo!eXABG;|s(Bj;@CbxTE;(KdWVE`6`+ zd#QKQWtHKaHbBb5H41%~?V4<^;g|1G6eSUQD^k6^mVSY{ehq>iNXLb+^i>X|A?wbI zV<~t`4-T~`s8{s2*Uf3l{{}feCcf-GcXFuwN|Jial6p>(dQ6_Y{)g+p2-?k`bC;TL z37nCiSdY%Y>2O<)QOvWCD1hlZ(Nds3G>L1YwO!ZNHXoZFL9})<_JX%s*y3a3tn%@h zKb~01!md*F(*zlU(3maKc5-=&eLv+jQ*m(ERf2AviH00AHrvm~w3LOpTKFT^QI7To z!q1)nz$)Zg33OsmwoP$MOL}IvIxehIs)x}?HKP%3TprbNwpgZ*ijUCxBbI^H3j`<5 zO;p-hlvq=2Idrq$I{xVsgZkytJ_MqdT$qVb--RCniA&aLQQfyefluP^m48?rEP|C0cQ*8DpHL15KO# zkR9D-1qDjgn*}K27#!4?r88S*26;o$*UQtk7k}1l{8)fmeKC_&nYL&#EoUX7CQ|s zT7^vfr!7)$JByH}SeD0_d`jM)S*(m)eC|d*su~f)au`fi`al)fkOhY5)EMfm!$65v zotLu10?Og6r9QpO=O4Ev)pE&ow%a~|M3u(gQh5<{B&8y59=jp>E;rL)VP0JyE@K}# ziIw5{c(bTgs>zY7Yjdfpm9dHEc3B3i&$fv|CHAD>ul2e{hP_^nwvS*@OQ!rkqBc+` zdR1PeXbs8O$8B0Mb7|sDo8=*Sqd_kVAZ~obu|ngI-N1B!zu^>UIj&+X2yf&}7xx$~ z0&n*Rpz=8F4@bhW!ciL6-^7^<_~Eg!i*GUYkH64ZzKTm`u9)Tj<_z!)T6qw=$Jjpl zw}4G45$SCZyBRTrc1H!8VImb!8Y=k2T~Q@4#FrO?npM0VQTyO4QNj=*^t?~#|1>7S z3}%(My#ksS%$b~BMfrd5UNcsQy_dfT%xH>R(>3`IJ*sKL&L+D~QB!y~6(?pM@Q_f( zkc6K`*wzGL;Q7vRY?6NlrFlld(}<@cz^hybXysR@aL*?+uqXvfAQ6eaE-8RU4of-y zg6zSk{X8BMly8CZ-OzfiVl+ud>e^lA?9wPBsnQ75;zbM`lbQjVXNHWcmBBF(6=ITn z3}4hA0F&g@VBYPEOYVnMZmi(bMlB{fiz;M}pWXpKi|SWFEzV~^qZBHHj3W{;JQwUw zWEF)&Es0kWsRO#Xa#Wr8(@dzKXrG0Acph`iAGk$nfV*tA0k4D7;4k7Vq3A=tH=TOV z22#w|I0HGsUEg_bfa94OTI>QkRi5K;YK+BZ^1GeraE`!~@YjqBGZZ1L5Qh^(BBMKW z@h;Twz`e=dX3T&S17JNG5JY>*cc`CGk)99B7o<<$KRdFg8{yvc0r*mz(SD*{m|Iw% zzO_vp(O#kdyh2$J5XkXytriM!%upxDV0)BoXp%aXPP!iCoQ|^?=r_=<-?Fg5r`%1I zx7$TXr&&KPfpMI1GOb;iPr^mldSu?!uzwFHYG|!*$(#`oR$NaSORo-+ugH^*mfUjl zJG2bECeuS!Qp<{pJFaDe!UC;y&=Z8ZwxX+V9et_bDlQN(e1ZvlD)uw;q+3h=UJq2rrrwH=vbYmaRLD=b{YAz?&SN_Cg_w{2+ zW)+S_4zXBfFlhL>E)9pm$uolVwRqjjWbs&yo?Sc4tbj3MTsPlasS%Ez*+sz_YzYztZJR1{SH^YZ9~ z;#0dAh3Q=|(zJl*z}8<-yeNi1Byr;&*sv6f=(S2h6os*Xd7JVlMz`)LilVg2$|4a1{W4Fx0<%itTD8E#_Vk^vaW0nqnOtw4;DEaYeBag9u zNUtW!Cw=e+scXLguA@hR>>XSD6fGSG$~YIZ47~{ThU6@}3?M(Ar7RXVY==qB>y-X1o#kV}`9emV zdZow%2oG8y{CouxB17xq4N2Js&}zf^}`Q6j>%7`98F_JKuar z5F^imoUU*qEg1jmv_HH0dnY)p%bqO%*>EG}^7tv%y3Nz-SW7xFzquc|I?fORkC1BGh`z@``xt{lKqLH-zx$p8CfX3H87F* zq~EH-gw8HR(jjAiovaZOsKXR;qfg95X4Lo~dQ^g&s(KODw(BXy2E$iLR1g7~nU`fr zZ{%D)5_+iTi%%84Xg8tDG22OxZXw3S^k=|^-o=cs046wj&J3fp`0925vMQdh?ZkH!0ME7r*O``}PQ=&usZ+dD=l=Oi z6rubNV-j@aG?QBT-rCfn*W_1TUg^Q(=leZA2LAMP4KaB5@4{swa7y;CGzLh+$iIIO zQmhInhZZfXKKylB|HUu7B+~j00t9xlva15S2+7s9^AF0SDirs|n&aQ$W$3h2wOFS7 za15fv3}X4-+@FsDu`jRI&W_Y@?0}_)5S%2j1E0LwsVs!(!jh?to1K$8l7l+35jY75 z0cExV=GV?4_biI3*@W5t->U)5f+3#6=DCV{jin_w(G;rKqUV?uay6cK)h7cR+f!cD z?pELM3+Kmn2{?nA0_iaq)m6Z2E&0&H&6~QKJW3C7xnq}?#Epk`q@{;m0D06dRr{6A zD5iMx!<$Y|`?I$#eF`@3WQsx7HiuA<0T~@^g)R&*q%22A!|CCphXG5FjOOr%IxG33 z;LCLsPU4gSoBc#U3IuX`teQ6^!O_QtUam{u4usw`^`YJ>s9y^>uYLa&^>F4=%RJi^ zAknCwv3zx+gWol>v(_ZrZ=e&j!JPRDYB&%tgqe68S@gds?B9-HXBUnNKagT8AkGIp z!LT9sC0ji{e((Q&x;`FcpV$dD$#{+bhr$-}G5Ehj*#7-i2bd}6m;HW|9_{aCB~+|f zGQQG_&3(G{cAh;u#$w06zr8EmikP45x)e2+MU54nWfQ`Jv>flUhAQ-2mm33CjrYkq zfFOHwHmi4VhUTx@2FPu?-WnrMU-w~YVU6((+k|JwNZc`UV9Q#a9qn>J+rp9#{q@Fu zY~mx|ykm>Gav#A)!!IS$G@E%J);Y!^K{kfm<&Je|zQp5aMs+4!h|5}gl}j1^fHTy* z$sZ4`qb~2To%;`-NL)<1tJw`8Mnb4|1|p5E z2~F5KzA)NLN@1wnXY;&O8#aF;+6N-(4ARhqIKfoYAwHfUL_KzlQAWwz8#JEECJYp;7;D){@ci6Y@k(&hWY zQ>2)8M6UTbkVXq|c>%E~n33j0;$^}@n(kKYz|RgI`im<{Rw@L9cUPOBUAvY1Y_f9; ztN@fJmL4-+S=wtF#7afUhgm}*;7ff|%$yiaxT;;@)%*!ZmPM%E@aQGLMfdZPZ4VI~ zf?X{wZaggaX?0(OA~voWX8xpt<(A+DVB3siED(r$W zTtH%;M;v9kn+AMWufb(h`o00Bhfp2;F#_0Hor-N5gLKGEZ&+4rV(sxQ^n&lAPclQt z5-7qV*1+kpT{QJDba+ z;uVc)r6)Zz))r$3xC7X~_8|kD{jzO+C5NCnu`rvNAl@|BH=LY37&A|ByRZFiT|nq< z-;=;5C5D^29A+4kAvWUyN3^6e=M&|_x+;HoETmHNQJ^tZT+}%Y$(SJB7SEYw7nmkA z6SHxr$*&nZx1+!fIDA?y=Y>V~5yJoI+Zl5cU+)oi*D_&<7M26s+g_yaPMo5Wa;x*qurx*v6OOwiSoI-5d1m5SN z*A*fBNOd4Y)G^Gp+We4B9%FLZNoHOhZNE;Gor`McM^DR%6Rd7z&t)$SwNBZpN#D$O z1`uE=630Ru43zY77MXM#d$gbSS#b%t1Z$Wp!&D>;A+xJgG7s~Y9W;adV1%9ri;fuQUt#M(|DK8C7bNDb=lSgXgz7#;4ebBz>L?%2cb!*F z*+CvIPAA46ijvYVsvpJ&ZWgxHL_XZ;5+5KogtEY!$++tAz*qoVT@NQzb)0gCZQxvD zHMFlQBRqR+%frjJn7O&ZySb&WEO(O*YX$xkWm8y@(^?4NI{J0oWL@?IsEdX<@(pZV z*!rJ;Nya4hQZ^0ZcZbYp<=2Ke*uA7|frzNo|KoF*LbGCsnqwtlgT~rGXg&Q7J8h#kP}OH0RWp&MR1-4U@5 zLo|I(iOutToLFL;?O&NGRoeiUxj0t<(rFX>v*Sv-&t8>XfAjvnubrbTPMXQt@w>q97I+-$K%3TcV-t7F zHMsujbkCJVZr0!>{&@SO`8XIGEo_EdHFH*9x;5EY*|Gxw0^W!2VoEq3LSuHdmdg^Q)H@g-*UQI+|Gl+eR)jlIS!r7q!Id7Q<>u^VkOX^;IAUO{dcT z3tj!q`TsyyPwH@JhSM*Z2|B=2Y?1piGsEOe8^}-R3LIE!D|apXLuB!8TMvAZEA;dT ze$CycAvm_XE6qc6*9h)GMUP!O#0xA|3)1^J{Cl2Jgy*wn+R&w8=SmmN7)=3d>S{}rq#ubfXY zo@gOP9f{t>v5CY?#w5fo0+X$%*nIjgXpMNPA`1jur~Lm0wH|mDN}@iv`2UHuu341DUu)k`NX)6cTC0dG}y_Xuh8F*0k%@NHFUA ztii1E$yrd{+q1l19Suo>MgKXjr7(fX1_G`1kU+V$*#&8al(Z zlt-RV6cE7|e(2eV+QFNyQxP$r?*B8Sm@l4Z&6Qp9aWKPofqPo1citF}^lOotO*N>> z74DB7sbP+u?hZV}pE<VQYtK|k=Z@pqT_5N@C@M-Fv&Ztu$`9z_#}$wU)HR0GlMugH&GzY84{WEL(6jeK?TEGHdyCp$ zEQ6e%!hOLL-$lz^LX_kRBKn1{D7W$1W`xZ<$(fFw>+|(#XMt^=SX4K{h z$=nwiW?$v#lnx7+*1b?`n=`^5OG7plV>)i4%XRyVQdwy%Nt2;pPALh*tl3GcDNjn8 zy4rcqVIaM^rAm)&MsB^MnV-VRkiOOMT38$>Ad?$Q#%j6$TVGr zP0sXebrXy^yo0bH1hu|Wt-xVBHHyT0dF!4V1?(;ku@;fN`JnMVm~en9Pa0p#?w$Qs zE1|hiEfuJsa;2xOA_)GU-Hom#6|JmoP&Ao%xp=(ub&S!@SPolk(g($KHstt(~ z5hA_E(R9|(Sy#@)lMh)5@h;SIX*=xpfTen9G<9u#BkWCrolJcnZ?{kVlACoy--rZF zR=R9L*jMn|QCND&{!)WSKWMAJ8xKCNo9700*?M@e=bLsFWNDP?b~qHRBnlQrArQhR z+Z-({lf3pe%=3zy0O5lLz|S}pA_@8{=JzdLl!14aP53S&m#OqaL7N$_3Q>U)QtnE! zy0dA2RsrK@OOSm!^puQePonV>VMUYMKY_dk)dkO9J?fEIa zzVB8a6u;CK(+vy++fM`tud6-lNA-;cL_}PoYy&|XL944KXBE(ph38Rbyf4St@HVv` zB}&vZ-6r8x0*yZch4gEhqmRcKzk`3w?9MFjJN76i?+|xCE}l@47_Co^F6^(}9agMa zJB!~GF}7tUsbgJ4Ld>%0w%TRA72g|eN@;ycs znq?=a)i^Tm`Qhb%K#3lQbS+SxmJ;G(t{q^b9^a6 z!PVYumaPc)tWx*;zoy1ABk|lK44PoWw733Rp{(kp$7}_%;%A(!%_Dv@bq)^vZ68>- zisH(sOcyCDUz?b2Yj3PPK;a}+L(q6JBCXv3Xdt}OJyVTu5#iyGIsK1bq68cq=5H+c zknz}xFLyd$C^HnUCYsy4vMP@I3VhXbENL z3b6>T1(a5hILM0l{9Ul(wC#I;!R3MN_a&k}=sX4#YyUSF1hOb8sN_A!FfR?$Vr&n#KS} z9aCUWH-qiTEotbD+d{&svYAn0cGxSs;2;~0^WUf{hd*Ias)@%Ab>$zs>7$t`ViU}c zkZ=$9TItceI zO5CyfN1A}da#Y1&-Qpvi^`zA4Iwq`u;4@>VBN+IycD5P|jIx++v(b+uM7^zC3wJ&( z2G68mmWMJ=r#YsbjUzYL%=M`ng7ITlGBnL_RcN@%bS=J=g{)1N{>_zEqN#zCuC+ev z6pXEq3u5Tn@i&xCU43J@gjJ+Dub*9*r5uXQYpkO!??~hSw?amxiOhs#56PCq4myj3Z?uXolh5PSEzFSR8VSA34)S6sD zIW2CunJm%D2Pw5JFzj}AkNN~iD?^h?-CAkOgxU%cNef%qf5!_jo1yy%4Y}xZ({L~~ z`SZ>x1}`{f#-l5jGdxptLDguXB!RVOPR|pg{@i(P85K<>s3Gj`>3Ixow4@ z%ZKzbm)MI257+M~3_xA%o zDDpO!T&`(!=i0;u7i%a~>fG#(%{RvPnz5vD>#<~8*}bWzCW4u~v?9DTV4NTg;t_NT zes5bE8n1QSq~0wzc(ZW3)(JX2`z+gUVKK^w^>u)nF1r& zpVi0X+n5^ka3nib*D532#Apij{Li4{5k;tOz}3OuqSmOzK5ERhP*12*&XvZ{X250b zl_{E*DkIN0TJTps!HojkY}$bM_YmVO(=nGZu@oVGpUh$nGZ6Cb>J6?7GN;YL_Sy`$ z1YH^;*tFn_NGr6Ak|eNzwN2x#CAP6xyJ7oFIyE*}JvIwx&;=JLOfv7QVZ_50IJBL} zE~HwCZ$L^#0+8**Ev;@~Alyc{f(~1lMdof7X*OUyroLrpdOUYChfZJixel*^sVKFl zI;V}jR=Rd~;^%u_1B^ChX;l@rIDIZAgwYBG>u%{kj2`kgwQ{wQ!PzaX0Aky_(&yZQ zg%I(+>sLDq*JdW9COsw!br~`PACF~nfW>7ucuz1oswL+?hHT0~@KF@Oj@-J!GtF}2 zfX_Lw!y(R!bU`-5eTFd53R5#HQBlo{HSUa`H7ECUIS_hI`6)rD6@$Z-kHfHT`zZ)U z#>4xLipU=~shPp}qrFU}e~B(;5^nRwoeczn@*DTzjMAOsjG>Uua#22P#qj7j%V)WlpT*d>(6 zj4e$3VnYt0yGc*Mp&a3{Uxr7QoN;x;bvmB9AG`8XRV|jAGgmK4U85=wYd54B(*rLx zXB~q$#WX@bXY`W)(|^K{I(WN~cNi;SK>B_~BDrRG}W9;zcymb$VK<6BLrvGH6)gf{ozg zY)btLk<_j1l?lUsmTfXl(-bKn05PDaozs|dYBvo|&PJ4$Or1@t|6tsm_xIqciNM`| z!Jr>YIUJ0-Lv8(vxQAtu=`~rcSFX-9GIgf1Cks;<^)(esT2ifdaN1LHKiVN9_!rvn zSsXJ9+Wp#PdLC|8Qsse&KV+~$wHYYqim7iki3_b<|JC|H>58Z zkMuP?{aJJZ-gGWq#SoF(u^PN%KAv!9 z5$i2>SU`R&pE&yN38em&D)^yS%PZ1m?^E$Cp2H9Eh4a}gXo&_EBs$RZ&go&7hZCU* zLPo$*nR{d^hC0E}w39u7y*LyTV5(aZV_;>*TC6MU`s~W-ewWpXs#vUOF5G` zB3J2wg?=qJhM(DUBM}i*QRMm5rj%gxoiTEmqSCY9a@$RSQ33HMh|VAKcINK(zTo%%hEp&d6a^|py+BMkFiQz+M%-h z^%n4A6Gainp$ArfQXI#Yag4A)o-Z4E-y6UL*$zm@-aLbxB`J2lkWK_ANo4cn7z#<1 zk}@_z$gu6jl?3Od1g8q(=bX7$NNJ;6ClI>nE`~U ze(@;hU%C4Pr=kHk!esw`IV!k<_Xw*0&IBwC-yWi!1waMU!Ljb3$m^#Vyrsu z<)CKdRtteN2<`zsRykqW1W+&_a_&T-+q)aZp}t%hta^w@V*Z6&XL!jwwO*<9i$zri z1TmDLrrT>v9;hyseC4up+vC*m&JBc`XvDcT3=TOjK@Z-^^&1Ta8^>un zFF#D}N_(ZyJA!QxWkUP2KBHpWdGL+W@q*o=gppJO=}0v`Tt-Ej+HxvSNT*T*@G^EiI zvoTMv%@kVd0u46PY;fVXC*t6Xf1&PnMnr+==*7iPxQm_CRQ(d{0#&E0D=hnBor%vt z8Qou$TU!&gqf63`qdGJ->>m5%gea@gKbodYZG>fC$i?&A;q8Sd0n+?`iLFI)4;v}G zC3^DRI5q`&5!V^SLGP0h)ZQ_%@!Eg5k5zw0X}y4ch7HbL;pb9K)<~mYIroQ*mm%Sw zY6CGXo^k(GSxhRxCSDg78b~?Ww0?T68d9uqGEJ?Exc!_BChAnbnC}+#`Nu<)H}I&5 zGXFVMwld%61eKZh!m#U3EVDsag%;|(ru7ut!B<|L{_t#7J5ePS|K2u^Q(TFrDlE#LRT-43x1oKZ)@O5fO4aHWEYZ6KN<|Dxh& z0*P|S30j)jtXNXKiQYY|=~^ul_6XHYQ-&`DAk{l6^jIedEq&Yah^EBOR%Zw{VoDSR zl2+b3HJP=3vDfFNLrOY|^-V#I9zKV$I*yRL1|Ry%P#a@E_x2pFs0JAZ#%AWih(_8%MA#%()|+ z=|=`=!9*OZM9LTme+(;sW!3dv;8>BR<}WAjc9yY?1Xxn$Q`i`%z*S)O2;Q0zXZ5i< z5vgAAXgGQ(irNB3RQpZagKuJRU8RNMyk<)&XsKNY*BN{wGzj~ zj6yX#MU_qeZCky1?|j#Y+SQQLE>OpZfnL7ZpF_W>u@<(&G>haYwnmj^e<{@db!fMiPF28XQ09}Yxh2Z71?N}jF@uSEl1 z!X$^ZBlPl=zp>jKPbrA2;v&dts-$OE8vCxBB$#H?GZle`DNoV8H8=7X+4lv(#P=29 zs$n9P+69Z~6 zmg%jg7>&vI6A}n-Qf7D^R^WyfUPMX>GLyQ(F*qTQNA!_fQpRMR!r*F3jP_8v_EU*+ zOr<@hHs5F<@qvX8wQ3t<ysI?+%&8zGwPS6Y&Z*TFX>&z31A9>BlOqc?xXEVBJl*Y(B z6mB3d(=CPSe39A?YiVR^OW{d#xK}f=*|~a6HO-_j?9M=dBz%TE>JqjD1klQk?K@;w z5f8JYDn<}OQ7-+MB+@Sv7gj@#m}F#oC`Ai@#vIqi%L7rcJk2@>qyhrtb2P!Rv@p!cc_{x{jCO18hTz_@H z_8CzOy-9?rZkDx%mI>k_E1S+^ORn1+}7XGo(-E~(_ib)LM*;^bQp&ix`l4p zeM4urZQNSEQ_NPgV~4oLmW7BxLq+>0e{WeYt6L3uU0bq1@35Pz;$)DAXx4-BfOB_5 ze{pRSvj{qnyj)-}6H~^7seOU#QOok&?B{VSpC{SwfLqY3ot&#?q$j&0pg}hip#Cz; zoh?E$t(`kOjwweK{4^i4S(?)mR`kbeH~tN1RfHjwg8C?6LHpZZ{90Iw;2snVPvGUM zhE=y|mGgo5QDzpmFymZQh^Vb3l}f7)}0X zeqnh~!NOx=k=D%3Ol>VSV{H0P=Zx83m*WHX)6D*oK8a=&Vqva+UvqrXNty|t0b+0y zU&<6y`GFOMfFt|>xo*OJEu?87lnZ{H??6jkCg*EU{^#4y3#r{%F!W&18w_h{Iiq<` zuh%FvkKW!ht7|I%0XC{^#&f4;c~TZZS;J1+hjsn-m$wi^f$x^)k0HG&T!4-2C0;GK zCf)4*PE11qQbWxxXX`q_dZ?*VC7~8*jJfzv!?s|wQ6)U0qM+>0MrGV06`GO(6ZGgu05_JxI~B;f)(x~*Fdrv5TN zLXGWW6{EE_q^444MUr*~o(VQO5}g*I@|H~)8~eEw)-yb(4e5kpq9W`q4?|do7_V1R?$1#cuqBLIAAKsU8e)US^}`@ic%4n=9C_0a;1>cBEz-p(syp$MesgqVMVK}#*A*j=jv~8`Fl_*V2 znQ5XP{Wal>_Ai%5Ii&Td%A4_=umeY0?2Cgh8#caG7@I`VKB>0m?#Hd0#ZYM@tAJ*&Ba>;jc5Q_(!2g^c{yb=H?Y?DfjuA zCT}EJ)1v*f{UeZbCW7fu&2`Is|N1rP9JM>E#t3$|nhw2JTwTTX_h`W6QMR z*EQm#2z0-4GDbZM5$e1a#lRDnu-dQ41%zLCSG1{8EI1J!l^n{R0#%qi%^-uS5~{8# z`nOn*xZ2mGB9t(TKoFb6t^QeJg~m36zr$=D5bKGDTM*o=-we|yt*!{@naW*uNQ4bo zKlmgk5g?b+YeZ=$b;q6Et{^(x_G6K#A1nSzsuj`rR2hFuRTiF)pG9e`%&CoSnt?3R z2Mq{t9|U|Mtz%(+R*K)~^c@i{vnJHCPX&GwEJc&i4TEVLo@u|}y#@|L|Lx3Uj}j2H z8@*r*dW74pvgNOPg?G8rx+QjrkD7~}1oDPILjk6dJzi+vfgkeMgFu0jF=*UJsr21(87ncz}s9otBN3F_^`JEjKY4_M|Lm#1mqDMp^I z%U0p?K0~vF9SFrfLF{nYSw%@Zrp#k)LT9A*Jg>5UnaJr&kq4din1)cQa4A(?~BII~_bT z705ysqhMOE0{pO2UjNW=@nl}DFPA&)eFkF9DXne9)v4z$#SX6YBK2hWPBhfkNG|IG zyf21XpwsBMN-ZI+gEZW^ORvnVK0>`h6#OuX5a>WV@w@`NV!2m4>50)pzA=%8dC>=X ze`oVzW%8n@^RhB{8L%wu*ToHO?YQ$pGGIf{V1rR;&(<4D@sR&q9FFY#F!6s)jE(Jd zXa0Qb`1#!*0Ds>No$1T@Irx2kKRzBF+U<0{y}#ct?Yy1Q`=O}%2KjwC-m^cS-^4{{ zW`7^)dxAG&q+4wYexcXoaOL43_neADniQr2f#))MC?@6YZCJv_x$ZKyb>zuw(_e17 zsdeB!t?_+>xvlZV?Vd;kHbbw%Bdxoy6B0SYHvNMp6kcqeW{u;dm$s?~lrl@}vCW@f z=KWG2fmS3CWE|ltDJWSW_@{{V7IGpA-cJ;0$tehpdLqiphYj4vM1!I}ZbyJceTKy zb*shLe&xLE%Gys6hvMux$xhV1_lF|@6{u)6$+^G1y(|VLQQGVU+^zr1L93Ds6NBekn zf`;-t3aTa+CkXVXrH#^G>j%LUf_}8E;rHL2HJQy^K&L=x1xZVP_(kU5mYBse&$PV6 zZrEja>v!n&1=!8J>2Sa02aG(O31fH`-5ze=CXk}?#BaCe{@f3B*CaDe^%xoVe1v&( z#|vL&h~djo2bPJUJob^JKa?)W#YrUH)6JYF5UjlMKsm|>ZYB`Ckc{#|IX-(JA7+EL zWu3H8j9MhCAb!({j`779Hagort2RK@$7z;BsgwYBqO5@BnUuXEU$&+y#hpy0b5*h^ z>7HDH(9k&35m0cY7trU~YI1~)Njix-R6CZ@CF!^nLygdiiYv?Y0`fQ;42)Ohx-9`X ztjK(S(Jl`f>`cZobN9QRT<%NSCI|)K1Az+lATypEM?|S^JgZ+E&ZrWgrr4BHf+kRV zRCH~?UEWJEk-d{8)0fkF6u`hu^~ram`v~-nLfFlxwqimQQF+eg3LKy)OoZXOE!sIQ zA+bpn>N@7cE<^9oGNJ#CchDM4#B{sBET)k~$`6J>*WeiR*z5TIQ_$|%Gd-+GAKg;& z>ZL`@gK!&_OmgQc#R4*Uo)9_)%0rEqatk-n$n^P3;5gmFia(SZxvC-5jUk{NSY#|0 zT$vKO^hWx!I%^07o^`NLgC?Z*Zbe$L&j)*p;fdn+Q$rw%`&#h_iU5U+Rs@a9X|wBGWS)ig?BJYXGbt) zV-=mAcRVwit^nO>26B_0RZ10a9aP7KCdUDiY6-3bN7C^Zuv>_k~bA(=2{n zX<&I);jhbhHORwur%|R=_07qJEnn`e7rB=~RKN_@<@J6D<7IpEMQG{rx24R^)p0s}iZsL z5FeWS$?V92R^0cDoHp=mIJ~CQo)LSgJ2l=*$X4f9J z^%t?p(HSm0Q9KH6O!0MFy*U(|@S}dHK`LuE_Yix>P+gBwHYg2@T&g)_?khV{+$g!C zw{^HPU%9x`Y4if2Hvq(uwwr}?!xJ*Fee;$ep{cs<$*AC#3ZoH#bm`Vbyq4Ra$EPdL zMDrRx@mt1&SbjkZ((I70$$4K?+Ah4B5}d?M*brcyswj<`Qd%MfH(X~h5{z37&A62l z!pr}iwCO2WO==)m%zc9|PZ=q&c*DTQ?>^C6QV!r}BNa~s0jXl+QE<&~aj5Su;3h)3 z1|&Fs?BePT_IDRh@?#nQFQM&<9u-LVo!7Ld_1XQ_)n|}HBSLoHoP`r+{1pyJ%(0`( zd92<$*^jD6w&|{;g3nMv&+7yxQkFkjE!Owa5{7}xrC6%#A7m3Oni}m-SW#pePj;=2 z*Z2hTk~K5!n%HPI%u1FvExC%=<;4-e+`s23^JBQqMy`6B(I)Kr5i~ude{*v!b^>?kSC%#}m z`H@=NU_Zi$I$86*HV?UTf}20Rn*(QwK1}2S=B?m|bKS;UPWW)(bO5-KEvCzA+UE=9 z3VDtW0cSb}mg#qlFyASkuDH}7@0*FW^tuI0XV!`2-YS!^rDvn!zaldaq*zp+(V>8# zuF6mk7_O*)K3#@e>UI{B;|2FZ^>~M^_puk86kmovhq9-rt$~Qx0?Sl#8In8X zT=y{1)un;I#FFn!D7`)$s6R{g@Fe%Fl zIH`R^bLn7^yy4&o6wnD;WbGL9Jj>)j+ z8!H1z!{S$ZeVtr(tWnjoLmUn6c=O92QFiN(#U?TX!2S^FFVeuyPAADhqNj_y0Rb0K z(5`GFFu=gj+!Kd2hdNyW)4P^}dBeG0>Siq5$iW1}x40IXDQn**K{MFF(Fl0mn0vLY z4pmLq7BvXZt^Af{5|_8HJFc1aKw%21drW(r#7@&ke4jhu+|7EV>@Pb&E~5c&LSuju z!1SFudkHo&ju493Wc#pZ`C?KNSSWUB5ffniVda3AR*#( z9t4A(N&m_c_JYD&@_H&bVks3+q=>FmX#I^V%qaey$erOy5< zl{wGBC{wS!ZYJ{9Cl(F5w+_+Qo?F0uaMYEVFt4r zmu@ByTSUXCRKe93yuiBgRn8i@RD|jw^q3g9Yu6Uejx6HuCd+R+pt8zu37k;LaA1k( zK#43NA;0tJqO`^WjXyT9ULrZE9tT*Ar1obaKN6Bm)@bbS6ao~Ycgo2Jm@GSBoy(#n z95AJ@37J&nhy_CBuO=;S)ry+0mueB5zX>8pzC9|Nu7RJCRp59S<9YZ3MRX1{Q)j&2qg zwdwki3EA93|fgT71UkTG1wr3rwc%i*#*{+_jw(daSVNr1M~16K>v`6GYQ@Ajr4_UJ>u zrrLo-1`cekgeTfjTfnfw)<9zTmT3l8|J&;_Gd^DDp8gse+gfH%SQmd zc~GDo)D<}-JcZ4JxXQ=(=QkiPRAS^UCfjm%{aAze0Y(S{r`SEJJAtb{2=vJ{Vv!^4 zy(XN9j+hoDB^oPM{rN=ff>d@OxguV*FL03j=hdU#KNjGV^}7bC2v9XB(1XzM-7n{~ zX9;-!P<=>J#zNGSAVuIbE>q}Kn4}$VXT$0ns1q_4^JQht&`C*`u~m;eeiXNL*kz{y z@Tc@o3{eG=5D35#No@(SgO%D-PQs1uj9&q=(_E8BNvb!^s>EUP9N^h#Z_G~9kbc|z z0o*J{KdM&doaQn>xmF)lco;8qW9hYG;3S<9B%Lo~nn3yWl6l5JpVS2yn}O+C1F zq;i`uAFO#@j=gT#6Z#|I3_((qr3G5X`c7)smJB+ z=SG0-nzNhMLdSClYKbj2p{AeXMB08CsZ;mQQ`@!`+=F0$NKo;?aD&RXwIiVmh;%gr z?VsjN)8W0i%{ziKj6cN+mPcn8CH=%i5G0@hdJIA3S<)D<{3W5OWK(*^)yYp z{-&UP>5k6s+g6s)P>q3qEM|4>TbN+VAz9gr#hGUoeb_y+3sk2J`mYqsmVYrlz+;A< zo?&FkG7Ka9{Kid>3w-geI~cFI4+Nd_DnHqBr|812E$Be6fvZo6aSUd%8PZJHm4m&(PIK(C@hs`)Ad$ z+}E`c8~UI!m-EVT$cnfn&OW}7iH}byZ*k*KP%Cd$l|fo5_i5X;DucPIAdRtt3Ekkf z@MsdYxkac04_*wDAwUbHfFbbuN?@pZGd_n1Lu}rC#&JCIJoW%SUtd`r>)qjV87i?$ z^5E#sMt(biezu(cUK*V;ao!nKfFje%n`ygvG{=_yh^qkrbFf7LDah1WqA8}d&aLYp z$(Sr9+IG|#B|@7aS^?~nA)p|mZ~+qJ3HIH>D*O(jV?T|5I5i{C1{ARw5b7@S2#gXS zmr$3r0lw4o8&{WtGqUnUc776aB*{UWb?CMXTH8Q-7Kn5sad>v2<70`ui;cK8~R6O*Xo!P#6>U%!esqoSY(Q)WF8{ZN3-xHuuYBS-nF5_hl6BaP<9A zZBo<*)Z7|`_hFyfa9`e!brF-MR_y*rSn`ruZ8y? zEWO#sqI;j%MqsqN_|sb(icS`f<(UeZY`ZinkOGqHW;+FaGdJf>1dPGi;f-{Y-FDqx zRWbGQsF`-FE>HebxnE#0aKNN5o3l@kh%4^Tj6DVrM&AQJIg}Le(94((je0&b zWegW*RujyP%eIqNRvbkREZyA!jYGNX2=x3?hb<-#j0n|R@&>`PT6EOcfvlBC^b#`a zdQ1)bmoZ+P`Jl4V2T`QKU+vLnTB%J{&a}Aw%g{TU;?LLb-#m2$tjgY9-!hbJ^^cqT z45uf^*iT#Em1kPf-Z}l{(`L2u%P)<)pkMOCvzxPrDOR@pf3L=jh8H$x7ZcuRU5xSS z)V)66NrpE!XNggq?D%_$Y#H}WwzZt=zgZv_R~rT&f&s2ym4=tl8uvqiwzPzkLrq!! zqyr%}{3Hr*+8j_Z!pU(KFd31Mx~$q=(VU@JF+w4Z6&jbW97*twzch|s)<#2n0!W5g{}LNF*#U$MtZeXS^5*$Ep50wH}IBfN%KXwep#NdQik1`U*O%D8CaG zo*}zK5W988$vLO*Wp7*_+)?1~%4Mbnh>+;Xa8#{5wbM6z)ywMs%uC@axt{07GLr6Tj$;Yi=5fFXUbYDcC3mG=)pB`0h z)dPyKpyw(l7AS^*j3HCGO0YBS_LHTzP0`87JtbM6?V6Aaxr#%?2HM60KnRW9?2AVi zf&N?*rWqilA$fKIx0;Ilg54NWW)cU(sOZ5kCgK5&(a*Yc0v7}PdrNH~ISOd9<%T#_ zrTtKIyOyXO{}M==Ykgp|wS{FsvbU$wmNaSe%2^eqD9F#OeSLFWHyCHoh%qWYUL3OY zFO`KETteA0!N9v%1CSEfv@Q`~W*w@rQlpBW?`wz%9In^oC73*A)x!YJHeF!td!1Hp zd+N@P-4*x8`vLY|tOVYjG;M{K%@J6(A{KA=^XD00u-0SbNi`J%Y|`n7ALD9M2u0i$ zM4~24Z<4oW^&?D9&VljNpvO0<)hl&au&k-Hcg!Uw$0ccrqvm7UfZekM_6us(wXs~ zV`KpzypRsV5F{0x_)&!RzuqM0dHM*HR+(f3s0Oqzv)#BbB2Xt6$Qoq{ThMt+HXsPg zAtUHZ#fZFX2Vt|(;+B(QXhvz{9JRpIO=bkPscWKSKIwJZ$r%>C*>WkiF5`%H^BMF0 zQwBjm)m70PF_Bo2yNG@L={H)p`OGHDmXB-~gYjEk9i7z%#H$|In#!0dMlV~}kSNr~ zG-tf&un3fVtNV-tG5^(NB<;ePL4LzZzkT@nVLQxzC|^ORlibchf{0`g1xUbTCOXa%mx_+&P$_0sJvAN^ zGeCtcJzF9)r&BdyUAws)E6gX@&5J&hOVG4AG?L|vj7_*5Y4h#*5vXQTzi_TBfpFq$q9FBfg{H&E7G?gNGR4NisIpQeZFC-i9PM&04bDLo znEJ;V*Sll=TW5nBFPkI$@&li0Jc?i8!*v<1FZjTn7k~FuMLE~WH+VUbY}VPUOn_p?dS0K&``kQftV;(E zxOmA*77zD0W&0dGWpGNx2kkx8ze>PIZ9Q?eN`wvBcu{m05C0WEkjX6?>9g>X@hu+e zGmB2%#f6WvV8)#z>ofY4aQveeX8)c%7QM@0SdoA&JTc$duZjO@6W@P?WQ_BjmU1O& znP=(nvzIBH1)4omGQhl!F?w3V5W5$WpmCk=k#v~{pjqr!BLlAWUdf|=1=CLV732eB zn=ci{N(W&^j}m`!hVk_p-AlL&<>OqLBR+)rhLZEOj_9P&&gW$a8FA7mIfNY6`(AVi zE-{37Hwh<5b@<9#3%)f+i12ZI<>$nj`;PZ7DFBca0EiU-O&3CYlM(+(hdE+Ey*JbQ zN*5RW@K8_69k=3!-8S1z_EA>O{S$ol_-4-MMer^oa=#ATZ^q?0+ihT8lJ_!1^dcj4 zxeh#dgh&-GCv>{5=9M8rxVSz_E_nD(BEgdj7jins^!`IAwJ7gaqOf?Fcg;h$Aa}IC ziELoD@!`v4oc$-H;?V-pg%6%3dlpdg^l$zd4?*C}X+X}SWP;NH+C@F06CbGN3<({7 zY+@t4mLG$2R+z|0dXp`3?H~tB7R3DRjsyp|v{$e&cKVz9S0ZX8>4~GqPNWgq)K+Jx z9sX;}vj-XcLQ$3ZN+!3u9SqQg^_j3k>8~vqUHVPiSpcQRJ5sPzn?>f|2E1bPD-LJssTiUz%Qm-6)UzwMah^0c2SG zU-g-6Jb{riWLtG~Dx+mJl3~!7yMl#cn4ri9sW{A1s_75yINzqaVBM_G`*5G46Z9yN z6SH^M9D&vy?zhAFxPz`d%PKRO8I1`oyPGs(masVTieUUeWdwNE1>>N_jeB9~8HdpY z{19c%3T~&QG-5h`)&gZBb`DLo4E|NWT2F5 zKUy`8Ce01B=;A^$sLUo>;|bT3Em;XuLnF)D^cGI-6U%e*+-$g8Nyw6CGTJsq^(*q$ zQ6o=Md4P6RWpjV)0Ql$^>%Zal{?`ulz9_ZgK7c9ceC!>3pX%~U5ES82CsGsBu-EM` zI3%ZxBN#*kOXuo7R|RddXzp%wo;Pc~xSP}`Z@0uczitnQ3jCjpS-84hZ*mUccy)ug za(5Iyyc3!&=WEW(2|iHeMAl22D@C6gAcCDTxz!7#tyBesT_!pvhRQp&u$c5BF_1tlFCQX1|_%P0LG)TXBsA@9mN@$ zJ!sXTFpC{%f~PVg!!&~NLEpUlmI|PWhiI75>IZUsTqmO%Q@|8z?Uk$8GjgkA9jrtQ z<14f|Dpr{G{THCH6>smKIc^*Gfi&v#1rKUln*#2qrmPq)e4={`6l@)y@^WgM3koLA z=H#xkUclx33x#w_L)zt_9KNjwwQf;2xC%Bb6{#||&O$}~lntD#)q%f0(N{n$&$5Nj z$a9X#hhHS?eI$lkW=1HBn4Tcc?-j->FjcHlmyy@hzE3tlcq21QLFJl6)e#!-RJ-eCAssPN zx@;E`QX5A%EC7=0a;Ml8K_x8^wadVaVbZSa+__G!?h90s+&@Ud_{Q_A(?`bhZ3A*i z#`N+*Fd6p5GU|@)>e+*NwUQ;$Cjex(q!hpj87Sjj;V#mvwOr($ixd&4o(Fu4N>n~~ zB+k6G^hx4#Z*Hu#3o|J~|J0-4S|j{9@v;k`#_esSxpsF~Db#L=X}*BjEFazfS}Ba> zi9*#UnX3>{BBa^30m)HwMzI%j@hCq^)Op14j#^R1?tQ7lIoWmd59o*<@F9T9VTl>2 zAs4=+OI%OYr!`8kT3A9Y z^elr?n4Kb4a;$TKDHA{Q#->$YEy_FDt;kBqBEzP;5)EttZj5A8Mi!}MRX0^IW`h-v z0R*_j)xf*VMrGZ~N>#mGIvT6rAlBL(%*Y?armNWfvFd0139>s;H($<~9+*_3G=4K| z-$6?NAB>-|u_0$lov>ew8fzh^5wzZ!<)O<00i7@NPdzh^@Bu1y)ST*fW_#-lpMM!P zK5oKDyp}(?h&JGLUg%TTv*BcrSviAiaGCee$HJT*Yhlz)2B3z)BI~1Dw8h1?3vPb} zRf}_mCy$O3?-dB-xL%x|HW>q8s|SwYGMv zd;w$FFwqrBV)CfciAEnZqE9br9iZd4vmKcz=f&kl>>s%7CPFvC1!qM#N0~p{G9mP@ z+%h3i2b@N=y^#G1Kg+Apgfs)P#8O81CosrLj(O_PfO)C&C=1Do>s6lNf z5rhcvXY#(8tqF;Mx+HRMKXkB%u76VyFryahUo+#X{Ef75aATS2>dG%+4}W+kH%*#L z!;z(v|EL=WbD3BT)Skp2jw1!CF+p3w4D>d1Y|mC+Y!r_!-Z-&c+NP0bq5W`t!i%~TaW9FdAq6`e3_9LDWKm_W3@UP{4PU^iD^K=3k6=-RsrO7M&D{j!FExk9=xo-E1iNZkCjPiQ*Y9>JoAByZ`fi zTH}NJSGz8ZuOkYTSx=BU<}+kH-3Z)NNB<1^0m@Nk-rj#0-6Dlm7f<30u%0P6&SoH8 zX?RS_!^E{SVOcyy*`#)g-4h6m!Ou|bDiiAmymZI^x7>mQqytSiyG{H#VcVR|2rAPdM#=CQF(bxa|t{vZNtCQi`3lO#bz?U-=JCBFgDgB(F zCkd*NL0&G>e3(Lhd>;7aCo|sf58c1gbmT(w_c1KF1&M$8NZJu-8ZqaTMpW`~DdIeB ze^VTKCcjVg-OdlED3x%Q3Ff*-+*9+t zl#o>lMQbo(pt0f9BQid=Lnft|#+Yn%vr>pL1M9d**?#90ExC@jAYU+CyOH*WnoH*u zhLJ*|Il&FW4%UY&pu~v~I^9yp$sz*)VzLskX+r(ti_LmUJRmu-Dz7Z) z@6r^>$N0m7Og$QW0%3S&hb5*q@mNt$ucsDSc&BBed~WWa|+*}CJhkkma%{6yA+ zZli8Apf!}{$>5$b9uOfjk%Yb>?)RUtuy%SR)2?=AsP6{@d-_rWT{(ulp+aprw|72< zhGSs^O*!>$b)3m_t5=D&9H6ZcKz(Y^+@$i_a*Zge`BSnw^o3H z#6SjNXrjty%!gvYlRH~|mN>mxS4kh!vDE8R3=ZSrjZNNVCNeARxxw>2g+FISiJYjP z58VZOn~GS(X_5JB+0labu{*~7o5`~{RIC41cI^0AQG%6jZAMILLzo3!0<-c~!?Y$^ zjxVApGvWAirpc2uHz$PtU~ee$Q?~kQbO!Ol32I$8_yc~+`W_;8Bk1(OH}i_NGMW~9 zYH4+NW|SMq6-*EmgC-ygnhAQr<5*hJ>0}z|va!40>-|Xg+DwJ8#*Kyj75BoE@U>#eX81vAceQ#kI?t<(sgI~c6AuIy`FN;$yByTPOL=s7wj_z z8XXfTgLayjdMVXZBRl!C2TdLX8<_5ti|99v1K3S|oS z-m50$rYF)hR@pW!17syf2Oy_IJ%mQ3zIptj=w=c}wW}~=puP`&nNea$Hb|w)O!H_E ze3TOgo4Lmgity-O>nog&1h|3GSB$Hf>1Fd?^2?*MIa1|kOmn%8M4xN?Zut_LlFeTE z^)~!|1HYT>p&un`v;A28FyBSBuP5n8EmlXMw1k+EGc+qXf}d<;?c~v3;Uqdu<;ndp__<0InRo4MPSd8!L+;L;9cj?h zZ)Jb{^9k8aVTs8OqlGeM|KUJ$plfk+zQhdq`)B2AMeT+fMOvL}jI56z_B5xHf#AxT z8-m$Q0gu?c3KTtrdi=A;o6_t7CUz`!2AW9`jP{P@YrnFa7q*@M_O%DfTRMO`)78G2 z4H#>E8Sw^mYKU2CHpQ+ALffguv*bG^L^|D*n5W(m;4OrO)3PCf_zv0Qc<8`NsXLHVmIz=#OfmLfP!b2JwP5FHbv2UN6%$$l$l z*63k|BQ|dgH&DAhFL;Yc42pa~z0|)ca*!fHr|JTk!kx`YQhnyN!5AA#bR%C%Q{tG8 za#P#RDE5wEWB^Zm``?ySJXVlbOyhvV&)kXI?%PsAT)5$ZkNsV(afN7Y_V+m+{Y&i| z{Y%i+{OdL^URBACeE#QBvKy)t!J%%DFj^a(SWWWR#}%zHSFCyXFI_3 z&%64i?(ZxUcg?-rrv}Cw^4B%$gT4M14sC?&W$`oxlEebRf@?D(;gH( z$O^z>17R98Gxl_`KqO`Z{l88II}(t@Pu3wEoY=L&7cj|f=NrS4jib%Y!Td2S%F#M; zjbv98=Ds8itr819!*U-7(rnpUo7RhKD68zH0H#K;>=8VWRsqDKUj*Y=;rHdm_QEu~ z8+;y59rj}ydUhCFwl%sYLpps|)lue$T>3^!dgB`KZ9OrR14?AlKjvsqeylyo&6Jw1 z;H#hV_HcasOSA}Le_=*M$AuXa9#H(T@%*2$f z+|K;guhtiY|=B8rU z_KO^*Iw1c)k}i7Juzu ztjDr?;#zzle;-}%6ujPsD=x1-+S%k3qRYL8YOEP2H_6#OfRDwkTDCw|80)t# z61^kEdip?9ys&h5P?+g#otno^ftt%N-_yi1cdK1@dr@S%75XCe`)3Il_tR=8TsD1< zrN~-plX^$S`b_PAP#W*YW!MOW?qjqM0cp1Z*HIHfw7i+7b1RDRgI_-%nv6qV_%qTM z`rhl<`3fP-hF2cdm4YITk+K^t-Cbd^APhiNS<<|9Z+6<@&97+t1Ch+w8bUhNi8CMN0)!R*x=%=H07|#oUlx zTY^7z&QEb^b%e>}6l3S{)y4bP8r=(ppGOShK$uTj#I+w+Rs!E4C9V2((+?yD2P*DjLe90Zc-bRGo&T zcv-qEJz~VHDAMwMX%Q&toJ?~g@KyPe0gmQ_W*X!<;bAU%%zzfD<+Yw#CJ28lqoT2E z^x=1RIPLl|f+(1z93LT9Vu+S{K-J{`Od(gFHvO^1c6O7Toc5_kD1qgcJCz(!KrAHj zf^HhVu2dp(l2K?Bn(?omAm>atJnc@aE?hy+s!asd0q)*$+qizY;NfQJIeICh=o1cU z#c?O!)0zn2Ol8s*#kMm;rTQPHJB;wr^sk6Epa)jqO#JFl7cQtz@xO>JW9v6}in7$B z)=TXC6L%D+E$U{}axpWnMBQqa5b);>!S^&&A*ef!-;dX4tHzE+yrG@%4t6%{jWt$Z zpM~Tjjggu1K(wE#kb-X^(VrbKQYRm#5CvdbI~?it2SK5C4ue_rW66A5fYFnqbtr zdI`=#poeCW3wWVAaO94@Yf&^LJaaRPgAF`^@X-+Naj~r|GZ zO0mu)6kq<}2mL4I;-ZHI1=`7^1#U2)zFmw{t6x5>7KWMbd$aCs04M&03ejCSsKTIx zKNw8f{w{yn^n2j`FhQ$&#T`%JhjwE&*QCG`%4>{6iiBzAJcu|~eVSBK==wN|$kl}y z$tERzm*dt`-(46v*tLOB3>yT09w4S1*0LQbJZ70Go9qeo42=7?NBpC8NjxQO%Y~x4 zn@vb%{`zrx`2F1A+s*EFaScCc+CH(Osc{VCwF}=OIvlcP0%Yw@4@OhhPDGlnr_{wE zjM{hBsxuh^x=o0lwiayKG?r>?s&4M;mB%cY+0Z*wRN1waH@ZgWDMh*X_y9FGMX>6# zQ9XxW4t&JZeo3@$uE|hHrmeVdE7Zf*aC-$puX~`3vZ#8N>X|5~^1NG-{l#W}_3E7K z+_0_+AUNgMPOscAq0@>({mEqT^#Sudpu)qqntSJA>5VHi!-p=%<#V|`*khg_vx2V7 z)(HH-?Oud2>ttyRY53;#8TR+AdU}S)_)em+vz`{EsCFMgaEG$FkEu{<1J=?=;Eqht z>$4A4fU5Dh&+0RT*f}a5t{U3m6w)r8R zNaoXwkLnt8ACM;Ik3>~+aYy;-lSqb*@GEN_ErjY!)c@EW<4%;SdL6J0D%gTUZPnFm zt3*ymmpb7AC=LG0FsFi&0x+ZvtKmE*BJ>2yr3u4_@LD_&g_@97!r7Q!h+#R_Y}YrJ zecjKnr!M71yFD#9qv+~YqV!=6Hn_NwI-do-d~~)xZ-1#KhnErZShJ}vdCVQ;UPT`# zki*_hrW_xw5Or+*K8}`c0lXlst<4h=8AFb^g3Oe_rR@c$DW}3F;!b*RtN%=$kzD*7 z6_#>7d(INKX&9bbPp6A#iKycR@jB~Qfzj@M?(QvrMQXFvi9>(*_c7~-2Grwum3niI zfYT=f71vnP$er1#S0HHM9e@M;yIpZgf>r@2(XW&CBW-qET2nE}Kb{wM3F=|BIKei> z-wK=`Gu6iMC~{ebEU6~A-eY|c0S1)9E{@a6h1Q5 z+QEBwb5i~dW69G<=my-D$-rLJ(iLh3MproqS<}?JjVH#U!xJcv)qKUS4I^aB3g6(dNg=InM>NLJ zwjn<#{ole4F)1y8Wo)b<8WTOw3^M;T$ua5B(1oXV;Y2J_p}e)3ksW!#)e=hmI+9WT zfSFM(9>L3oJO0AfB(o(j4hz*BIxI9r*hzc4&?o(eg;=lfthC#izH8@>vTZa3LzO7y zpz2KFyBoX%)#x{7UoGw&p0|P5Te=&UOt;aqy+*Ih%vNqYr?-R1a;+To!GA^&b*O$u7>8ZvoI1nC$Uzdu+MXWE3`@N_Lt$M#55#W1 z_Mov|2J4v6Z6jHx9z$SJgF^f4;=N+!d-oZmy|@vr zE~9j67?<2FTvr=CMwK z$UU5}tKL#Xvfn(;QvV+IjNi3QAlyGVGO8jxzo{KrNW+}l+{C`0*6?mS`?H5lN zX};vA=9)}q;$epIj_J@vmCEmN8@*uJ#238*I+DEk)W+JFSco*@sJXO@zO|4#ao41j zz#DQk=!@fK@@6an&ob!n3Nd3DDUS-e>EBg%g;;2WRFW7FDh zOA`OognFll)q?9@a+n(qL>Og>pbps&Zvga)KzvP@fx^Tr#3T53nwx<=m&QY5*ESSv6xfaD0+}(mpqT`)$(i^AKNr=LE{W zx-A66CE=NF7I^PzKgC&*lguGGsO)5DyNOfJP6~R({EYojGNKiChjW1_)i5-$dfk>a z3eiq(!j$BYJ-X^?RZ3M<=VqdWbuAKNO+i*Eh}8=Ymxb%1Dcw;&uRg`oupaV~=nv>I zA3qS#jr^Jgl+?HYRE3AGyXs*g05s`ugb2W13ZPe*3{s;WoK(Y0E!@jKRZs@>`U=T& z^8^}ha}aZl>?^>UojB{7?`ZfX4VLgO7G((%H?2|2k;G+RszKh8N(oehGePbY8T^LZ zcFo%`uB%0eI(Svpm8!m_DJg(l_GXfz%b?;cKYILYskl`VvRZlaZh+6-<;*CZobY@HyH(#80_rW72orX7^>h;my-dyNG%wfTxJm&Lbof|nHylc<_ zvRB*zvRmc>Gn#?_>=W>TA)-AXjS)0Tl=>C$smy1-IrT zRX)|)L9H&nnC4677H3h1FoQP{Z!GZs?>;kvQ1i&G5&sT#VZT8+wuPrAV?XmL|e_!$HRqbH8MbRF6HTMOxj2`LAn9;wz)^r1dCt zHFOe{uM|pIxt|FzMMN4ILpxWUBtR%nuC!}NO{dmw2n-8eYPErZDu1|X^Ff)D5EKF) zK(oyWsr=a;bauHSMKR?dxpK0|>X@JMKp;K{d!JjQ7lInS+e2Il04(cKsOgZzRvs57 zr?H=QGSFqfM8IWDGMp-(=Np)pfD3ksZ9Li91X5hj9MXKz1lW;SQQFpc%Fe13%+<_F zVK3;;jgk1{?6h{i+Wl~LMRPcy+V-jFqZ!hrRW|N@O}MyFkOTrFG{}o#6+Jm9KD*0% zEpLQE%d|he=%R;@!KUls56el0TigB{QPe5XAj7cL?Wh3&!by&3;SVe{2F`Rva1&%w z`>I~1#(1wm0Mp=J>{n-32gF?=h^8T_2AXw)cY6TFBnPW<_wUq{6 z;S8S^S8MT)omtjZO-hVlv@b96Za(`9GMR4d_&Ig{NcMGFB~_O^F}boDMnTiF2abi~ z<5?w66am#a^cK6s<7Ci|vYW-X5hp5!9aYccDSD8@i1YVeO<&lVKCRw{RYZNJrOlor zQK1{|ocSMmtxEsAdk%I^JruK~4&}9aHPwmxUH+MQCO`Fjeav*O(aU??+DgM*B~2-X zON$Hw#?{R#kDX3sa({$S4Dh9%s4|jt;G!EP@kzb8S+hQ~`>f%Y8 zB0C5XKmd~;Ppk{*El?JqxUK%rBxe;pW33A^Rt_pFpIP;G*LBZ$I-{2#RLA``REySh z6Y`eU`g!Tx78uMK;@86#wF(@x@oX;}eWc1Hl3_%}4mDSWxbEZ34YqNT136=LP-Y6z z+xE_qve&S08`MDClc)lMqVTT8zf-AxjiD=5ubN}f9 z)av>c_fNoXRe+&Kd*|LH*-3%d{lPiGdgCR_l;b3zDivVGzHm|EF9=&jlW_6Ct(@c;Tdf##Jty;)eVH zl<}Ste};w9=5uys?9FbhgVGn z2u*$98^HkO9#Pai1_#CsS{YdG2#N%3K?;Cwt*>ntVrFRn+gm5rk}e2{d7Hh`BBq7y$Q!9S zdW+)t_{dJ;xgpypO8aNGF9W^SN{}IAc|2N1$ndYzZFj;s#9=uy@f4v%mMKdgDZ|VW z03?>T3;-)Izrc!Q^FH!O{)b8-O3i=y@b+=5c|ii~xB26`J`s`;&hAG@td0^=bBlM;v*ExjeetdbSBrfMXEWO2?84}+5 zkamP-xEZQ%6kiFmK=dOTmZ4NiHzpY8LtdpTd#j->F=(|bC#pAKD|DM7K&s4S= zQ%4&N?D}i+<%#4+P#cjRH8bjYLIPJfdT?t?Jb2l3*LeFEE-Z?Tw27f=Fr*kV>8Qf1 zsTk_lO)8<=K#{9s@D|f`fQC{GPjvvB=O%K_2cmfr=i_(@L1)}KGT`ScvjiA!@AU^E zzQB`nHbbKAi?J(#PR1lIdjhZ9y{RTD#EeIL!=25zR%6vlcKoLzqPDT%h;~(5QylS$ zokdiN&gUHFNwa&5Vs<>jG`h?2DRx&oKQX{`djG7Pp-KJ&YjyyrhO5zfFF#C)heJNW z72c(8pfG>z22u)C(5FTW1-ZP+h}UzIySHmDpBLy)el;JE-aleU*j>>t)N7NaHO}8M zvz4dXJoI(uE=yNiN8ir7oAT5RMy8Vja!F<)CgC=i+X+@}Nk3oKN_PZvXP#CcYjbWx ztSFM2BtW*-__zCSr`L4i$Q;dV05FNNzX-IcyD*Xcl3OWDl2O+dtUL7 zIbtiTf=*-Cn7K!tKmtAxv)>*v+f`7cvmCQ?uI(FuVEd79&r!~4&0^c|>HTdz0sO%LTU7Ba7P%|@M>Rta>Lkhw7Y=Jy@G zspH8e|LiYuSSw7_dtZyYmZOj~*&f$z`M(4t8!T=H7?_}etXv;VBmyqz`KygTRK3icwBu|NW<2v4?4TW-)|1$sN7`jl3p8JI>Vl=_JgpeJnTZlX?G{T-4 zMgUWtR?Ajm)@9;O9cVif&>sxu0BlN#Bojl*a*^>*C*yDxYVY4Vm;eTlJx~?HnkS+9 zcI3{xYDA|1T7Xm4Jl)Rp(qmZB>>i1H4eUuAhYMFl(5bCr3CndTHdXpDEriSlgNk8u zQWgz$wYCD4hzbfb0U}@*`Me7uzX@1h&DM zO9%~O1_;SSSX8SM2a-CfS4Zid`-+jfsp4Ij4yHgJdIPl-F<%GYjs(P>$NgK6y;)=) zZjlm!8L5ng#8@+EnMb?s{2r)E*x>+FSRv;s<2@EQjXMPY!-wyvc|HcqEgB*Mkd#oY zYBvnwW5GMu`JWQ6RqO|;l*Fb;Cf(I8Ho6p^0U{gYVPX0Yml*9t7UE9skTuXR|A<6t zH2=zISEJVoBzKOC&d|*c)8ZN<;ro#&^Y$hrgjkp(>ejz2Wec1M^R5YSO-<~X?Y*OH zj*+iOlkmL>7Fm@}9kp%qqS^Yv9+p@zu5UoWqGw=4Fb&(V*R`G7-Y-pf%Z3kIYK*~z z(*_ccW{BD1!7?3Y1*J?gg$Fsjk00AcZ|5padbaqMWnOh^sjxBk0ZMG|=?;UihpJCe zpp|R;No-hizDegquXpXpgAyOtY z1T^jc$Ri+h$2s%^%~+l!Q}^F=QAn}^33Qgh)xEXnP1o=p#-72?w7Al>w8_&02MQFT zX=LtW3`@_Mm==Zkn}4ZmuG;+%wc0}RU#I`l(EfDy`wwb$=v-}x5K;4g0mdyk((E2f z1&wB!@o=pkLm02lQO5K;Ez$*Ie<8vx?(O|p1VM8X^yBo;HT>aZ0`K-;GIIcHz_+-i zCb-;TA05$$HI;t@rQEEK(B-e2yXO4z0!J3#?`t^PTPy698?MrMK~=d$BhRccDxb<5 z+5F@nn6aXg;nUbkSQ6)#vyR^8u0*d$BwSzpOrB*-PO*C$vkc3qIcv*jI?MPV6@Vk+ z17^eU#ZkLUIM~Q&mit9>lO`8@csa$eTHTBfLy-cM18YE(VfLMuV8Lu*TAC5QG8e2A zbMR3{4WV`vSxI?e!WetfYhu@nq@5Yjw9SS2~wAJ zDn<1b2yAi75g2T&-LuH3wM?xlkznK|FSlWd>V~ikh6oLYw>S}Rg${(ksNjhNZn-Vq zYevVa9Rn7g)Lc~_t00nw0z88mN=czWBaXxH1TOXw*~0mp1Piny%AI-Hq+ng_syCOwO)rpZCXDym z!AOvDqZlJRH|hu#B}w_pUTeYcRXkb4V_sg?=&gzq_-W_t}+YvFl%-g zdHLFw4U08<8SuuLIe&L7%gld=IAm1bOXuA9T3{MJ*UkcH^J-2~C5ysZ_4CvH<_7T_ z>XvB;{c-s`W^2|=Fhxu7Dq0X;xVG8@iLtcdGf@W`*=Th0c5keE$9@|*J}z*N-Pw!W zG(ZA7CEsh@3eVMjK2(&3Ye8I4$&KvzJS#Rq&!N8PlTVxL(iqbVCAEI*esbX9J?WB9 zy5y5C89cvMU6RyX<)P2!+feFh^0S@wnLckq-pbOt>p>W$z4kQmY#G9#(9)Qn}we?q7kZq({CNPbNorRn`uZmPDt~JQ>INDURWr0%VniqjC@t%m8wgt z@oo6pmr8f>t`KgaQh!>bdeC*7$XNA}HB2 z8Iz~UZoXy-OX%-Ns)h!cpt zBkUu11#nQmSw1r9b17TCwHN{zei8+_?4I+tFXNB)gt#+u4`xNk zxh0aW8RH82Z-7_$NI!32CO2$5v!S%M-no5w!Z0uzy{=DYnj43RZ9Y?}4xgw}YT?sY zRt?t?>MBi}J8}L^x6qKBY!)M`K2y-hJ~uk?dv=dym_%}6pUu3}8@4^x7J*3$z2Su` zCJtwKDna1$9s+D#hp5S3h#7e+OY@&skTw7W_fj(bR+jL|qpUnm=rPybI-EGTf*|-5 zp&THk)UK}!S@V0uHMtN(D!XKRkhbj>J9Jj^+MC+dqum#&8ryQP&TYqFE5mQV?gx&0 z7le;R9cjtdk&y%SlQXpj2(cupNLu|&Wc+rP`;p(&AmsYFXE?HZrdR;+8Lzn3eA`h} z^iL*hhx;tz0hSM>oo^wH<#F`VJ!ouSRu#+*MUb2phE4r%BWb{C{y$JESTD#q(Oj%E z*7(j^?Sp0}bPY~5LzlNSW0=rJg)depS37d%%+?MzH@^>uI=W%oquB(2G16duM;GF# z2Aga!FfV|C2mJ`XJL(oqr-_^osn6UuG|(DQUt|nR(uTedp@7zi^eq_PRDJ29b9 z@vE-8FZGe>ONj?1IN6j4vv z<~(>T*Yg&t7CLPLq#z-?ClECj7vzV^kOzvvVPJypF~(#BZYiTmlwssfDVt-2jA4W> ziUR2+NEboTde4=+gTw$6X(=mVNywoo-G3p>%v?&f@7fYykVwQ=W_~sqI(B}IfEjRB zD_f0f+kGmb77{`0oe|4%&AXQ*rr^DMY&;_+mkdvUE@{bTSZi>V1=S`P$z{q{HWuqD z@aN4i2i~T!hj6x0LzuF3_rkTB*PLPwESyE!9^27k-R}Nw=6F9Z{~q_pSe+U7?0;bZ zfVjhrYCsg$k@okvr{4oxlPmJNuDi%J*toevl`;o`izKV88ydNk;TaFKs(W3-^-rB# z<6`uEd`hfkxQhWoP`Rhb=oOwY`$(BHMy;H~?hL#T(bwoJtKNM2#=fkQrWx!tUn)jk zY^V?{3$Sp}SJOLYT9!1~Am66M@S%*EJ8~m+NphNOxbU!n!Xp~vX7)BZ02vr#Z+(z5 zu9kSiI;f56u7hoOOwgU**Tu;!lYoUowG6Hzcz3w<{<^Z{BKIyfE`l4D*&mpjJCHMC zs)29h9Z4m*V-PuGgg}Vy)=roVW(cW7iOdTnbMB*zEme#jgOSsjRf8pCnkdg%CCm=i zRW@F~ASG36R+@H*sPZLKLvYWBEyVO-Weyv29H~)A-nm~v3I<3_D>){yU>KpRs)(hx z%+|lR*J34x<(RnghqyOC{O~ruT-9;?o4&2SlPXK^cse{oh^r(grlHdzl8CUHPAaMS zAVAlI+}-gseicUu4H|2%ty4wK+7h$4gr`iDhMy=+PT&{A*afdRB9=khP<10;a#6$L z)%$8L5_4XiL0z*Ffs-I}U6vc7d0u5D6--uoxJ31@V*AE#`;2|Wqjz6e*1DypU;v3c zH8f_lWKd@`re|JC@K|K*BTFnWM3Gf%-is*o0HyBxYuVSux+~2FK^Td{^BTtp{B)(~ zf1NVRb$8kMlod>*Q2pw4O{%yi_g;QmwdvDW**$jkqZ(IOpl)wumTJO2mTI_wX(IF7 zuuI?Gicql~tlt-<={%0;lKy%{7c6Q&l8-tGyWTa}T%yLuJ4l6{wfM)BUCo5%P=_i+ z9xf#mX!aP0z^?7V(+wDb!N$=f%2!0l*670I8Sj>FyfQ-R);X0UNH-ObS&a?MATbsH ztviZM<7|dk#m;o#rPwH-n1nA&`p?PHSb3kR)~mO{P3N+UBC7B=AQu=^wBxr7ASWFn zqY=T<3T=BX$-kMvACu-A*wpJ_i3_1i3OV2~a&x7bJII1~TB<@XfXgJ0=qNWgR>G7a zoNP<(l#B^7LLH>1?fZpfkjZubOxD2Gg9&NFQctawCBoOrfxaAJF|&pbF&n4 zo2q;U%FEgB;6IV`d8uSUKA0|-70*(po?8<9V4)?woqxV&0*vtDxCA73!11(WUh{HO zEfrOegJMm6G zd|S?DwsNx*h=e7KDS`n&xiy>p?prtq03VV_kd$P1U-6%$WfE_k_c_0_KYja`Z^DX5 z28!RUgrbi#c;8s>cThh!3}nc@TeI5PAFXg+vc_$<*ln@WSxR0KvkaYZ9oR$A!1d&k z7s-`*+Q4jrM0%>JV!F-b+=T>|nlzNGObzXyz=O%4DvB!zOGf`@HH5VC=ADpkZJL?K z+`+N9xMplWHoF+L6k`s$0Cq$_{&;;Et^pO_I3)r8zq%ZFA{wU$o=5&$URL^{6u&<> zwyj~aY`sXch!viJORnyRYQt*{R`1>S7gxmoT2+Uu2Jx-UcQceXW9SUTxtT?*aP7ui z-9JIgdtC!{coQYan3c%MgDnOgpxxf`>=n66RW4b9FjVix2&OyHb)jUudUX&h{eg~y zUB)Nn63D33prh~V1o{RYl`Q>`4n2a`cP@9T8nwZdZ=eW;045ZA%wBtF6>=7{<(Z1< z%TpMhOQEjLg(~tz8@By~FL**_vZ}yo7i}1Z=sJUF!WUnV6Ct73h^8cAt}K`(BrBBN zpFf$?yjvGf7VlDa!PZLbwu0AjS+F6V8s0P}KU(f#*)s09SpfIOQ(5((i@i6aQ_Xxd z*hECuoo-KTrJK!H?FtW`#NKim*B@9W21{qyOwCcjwfR9P4tT0m6N>aH$`fkzCUi$^ z-;c|yn-5nP=eJjvugLed2VNG{$KT zOngg3Y9p&mtJZxoNzF?c*mKJu=MmmD#&`w;7h3idV5$@-1>#YqYv96(-;_H95H3iZ_E7}=?wZ+&{ zu4b0yc4vM}XCJQ4FW+8`7u)tNgnF?w&IC`JYix9(UcNzh;@l4 z?qJXzR_RNOpC8dtGpJagmZj&rQ?T^htyZxlUb)aMiHuKZRSqhmt}PW>3*bsRgtG#} zD`RrOF>h?&vg_Vff2i2%@8)*Fmg;SFuv@9FA%8aw*CKS|T{~5pn@KkE05Nwwfn=b9 z>6b|G)`p#d(OD_+>tt0oG!|kZdIPw!g?yW^?L73Lweipy0=!7;Cd(vW@`TL*29>8X zRn%d#F2Ai!L*5lDSd8~oyafkd0W}U6f_`&`H9)_LKcyhJksQ0E<9i>%(>WUxZfBBXWu$AZ>FMQK?ZwBHSwpKD(ME={TWDz;Yxo(jzFGr&8Zf0mp+(Equ!{LD=ijy%fNyQ zZ$NNKtF*G#V3F?)8XT~@iNuHKv7M#N^?$#ThRo6769lKoNVDHSF zXPfieSDxMt{EfWfN@KzTz6=qvn9XD)xErGb^Xwlqj^!)l##ToWvN@)iD-Xm$Qy&>{ zqF0%PkGj3MfmRgx_L~9!xY^ z_c(k&Afsq@oT-LwPl@UfQ6KAdP_v5|3yEL`)lTi5GLE7rO+T4m1} z2{>qE{jlv_8i=nid*4kV^@EUI>Q+@hsfEYcztxlAw=Th_b^RuA{b~rU+pghLx`l_o zeS2f2pKv)Myc8X%lKa3KWz60cG>L*)u5{b>O=sL_dea&y{s{%V!^n(#8=Yl{X_zEt zS43JyO|%7twpX}EY6hdmm;vs#YGA8FY+oExTGOq9-w$-=plyg-y2_d<*$~hxb zu2lv7pkyF0bkj!)7-C`whixfP6iA7PG))}rGSd%?rR2qn7ifHa@#jDPnE<3@%p$&6 ze-YmL?=N3AclW=~-@dV+=n5-DVz00#<%ZQM`Ja>b?}~Ef)xW|~@XOL7E0+Ro%O*{# z=di+n9{{eT5@v)Y04)JZP-~0N=CFhu+!J>RL=AH@H860yifw@hesgPRN@%Xd2mn=FQpNZW ze+XzBP!LW5$-oOs>z$Rhzl?nl)(J8_$*}- ztuuhDS;OcI8~fi-mNu-yTzW<5YzCNBOGEbLf+qZ5MPc7LBR5xX!N&@ctr4d*YtDJa z$7eH}lT%2uV?wT!&`__|v}_k&c~}mJ!5U2y5jEE}@k?ZJt_V2O!9`VYVdPPvv;S`<{f z?g1ScN`yc=&>r-OIK9njn(pZacnW7|(g9ewFMu2N0m9n){ZWnx>u8SLwsy7&LFt`_ zg0^8K2qbpHz3~y8OplF>qjQ=hs}cKXXsbE=z@H%>+hO~#U|dN!w3%cYONV`d^GF3G zQ+aaI;W6(Bk$C_#Yd=)xq4>ZDMP@z>HuHcz=!4HZdq##Ppswv$FV#HxET{?s@Y1le0&} zaeh=J=f}r#?r|GBe=rQuIV=YOpJ8gKJ6RiSIn$NHJ+SKFy0#eD8R(QH{S(lU$_jVb z&UL(?!XZ%XaTyKMmpMBnu}Gh3YkJ8a)Sf(&e8^O@=dpH4Q-q9|ox&b|$T$kVkx0Jt%{AFvI<;Hfjam5L_Jc?6}OOr!Q) zw|jo+L(K%Gtx(K}957rcXJO{Sl(LZi3o7qf?8TxnxiJ%VwuHdmq+?eQn?_fa0=8x- z&&K0twm{Z;IoqI4IvEZscj_(86q-AXd;-Qz0kxI&+`6X1mC2xx?c(WU((P!@-Db+Q z^R#EXi04RRR;+jd%pY{KtVEDH z-aX4o@cPlI5=pK@D(h}*sLYdKu+Fp<&WP$wmbe*64#;M@w%3E`cC{*Zu_y=1{P^^= zn}u7D2V0N3*v&#Kc)0ibT%mr|Jn%VOO%HqyR@uXK*RQ^ZU7WpC`M~G#YJHUF_GBx5 z;Bz=@tozi1Lb=kB6@4jiZG|L;yMtoq+^ATRGE6z{W)|#3Fo*aIT-g5k&2>jpOXpRBp%8m%a$uq$#|Oe>(6aPb7op#2ruwV}N>B32P|5-e`i;IT0iq7d2w;kL_9 z5z87D<4k%g$IyODd9nIa1*revq$X=jMmtsbsPPf5Y)||O9g#Fmg~mXiQ*>1{WY=tZ z&{0#$7*AuvoG?J@l1q@nmr+`9uU zOTX|Kzm)rqcNy-s$LEkHvc9IVQ!o8g&0U>3Kbpplrm^A``Oee|a_@KkFhq;5f(L`km{$(YJ)9k%$3E62>h%4$_Fe zh1sKQGh8^-L$IcGY83rQFuyJn?8U2G2KDQqVrWNlUWlq5OT#Xs#}Kk|Xp!Tc@CCPl z;D+uo)hYLbkP~cR9A){bZEUI zUevn!b+54R%0^Bsdq#JK$aR)$%YZs>rM-9#ea2t7Q?FHYZqjHWFT?E3#gkF3nM_}i z|8w#mfBb#){I4e`Kff6L<(EI6{Kpvn=f6Gw>+|2;KmPdq`N_%8e|!7=?akFM{Q2)c zr}^R@|MmOH&+O`#z($@w|LcG4IM9mp%u_uQa)ev@3c@5iHRqOl*u5_Jd4Ww78ah}! zsBsof9Fbr>I(Z|P{!#gDcabL}5vs<+pUEDiOk*>RtA*qp^O zIL6E!IA#vl{TMJ8@*D%^jsbJWfVrbpaNBdV3c_;?m^%i{ZF~1KQbmpdbH{+WW5C>J z8!(4=VMEwlH-)x{nfshV=9D>v5j&3}k*E4r+_Qi%n3|zwtpz{|s~cWI3@u=qS#NH? z(R4JAIZH!rto;bO>!XY=o5n0I{$Q!SHWMC`b0S&9Gj7XJQ2afxGdP~M4D`wzT9({- z((B7Z%aUEeN<6i}gtB)tG?b5ls6KGTh?a$r(;hovdlRihaX zf$i}l?nZ>z$Bauj$rBcdG~TPpZmqLIhz6e{mNfEJIhQoyvEkP)?hLO_=&{6G=1-j} zc8bs}lY&Nbrb=T%&Cb@~Z!aodezGK=@f1}+XxT|6Lc4HGJNWvrZcAO7&jUL|!CT7} z0MLkXM>s=%SN?}0!mu<%9Q^hL$(W4r`UM+X6;$OkeI-WQF&txi7hio%<|3DBTn6ZR z3BQHzUK$tiOPVB%DotL#APb)6I<&$=Ndx`n>khXkT=(o*6C(J=lr>HGf3I#@OD)3k zWr8CyEi0dvqjaTqUVNd zWRIB?d1En%E*AkBbA}Y>TCt_Mo1MH4B?|lr*iP%#KP9L{x1X-a`r+Xc!-RUtW?X5x z!$raNs;Ar5uo99i@cFC*q8MAPQbIW8N4t=4AgcC>UifK7NuITmG24DD$)Uph#Z+F7yq0DFhB zc9(1G3ytD^E4!A{FchDLQzNHwmBxLbu~6hYC@^=)Kv(F7x!eimlYOL%EMaPV|1H?> z?0q|v)A;Pe)%oSytMOvnU4ClFe-+4YT%wi8$pgizmjS4Nw>*1Au2Pjt`~9YF3=P@| zIvK9~s{cGmZos!Gdh+_M!#X6|S`Y;zTD zR12^f!|z6n`S9K4kE+d5RT9n5n2ydOmd(#>>z~59E=0<;km0aTEeq?~L$(``~JuBom_1rP2xHUx=hi0^pZ+3Q#)Za@w6lKf%|NB_qcI5v^GToAC)D8UKHV8(Jl2@JPdd^bKBebO3fZ#HHCf9eUb;T_nKCxs{ z@|YP);N#zn``ntHZ^z(nG_n8y-Ts6yfHkf4i8@W&`qi=28K-L#rDOozNL! z@Q<^YEAwA6nY;lXe4biUN8>8MIk$}47kEH+F`SAchkVdpL%-=_sx5uvq#eh^CUzy@Z&WD} zaoUFBegoPWbjcAP7JSD>E8-eVE0-m&%XUSiw+M$>%T)$Ya; z6QMh2Wyfk*7XD^~5`gezBFhsDMfMBbDKMdncN+U2pt_gxcXxHV%PGO#^R(T82ff|s z=(f2)eg80N+Pg~2ZnlF|Qno89-ivR({xrt-%2feuFrCasw{h}#S#$v z(6R1HU_F)abibh5DZ8qe(joIBtU5~f^q50bLV1+>Cy!V4WN)=j4pHG`&vi|@RWs?5 zM;4Yt*8PI)u}aCn+9W%xNV1DMB*COCEP<`P(nvsh+v8`fg_S4hSd^eBH_##xc82fi z;%wQaEu(00qy5I!X>Vd_Zq(4s34QD!{2USz)rx`>Z1;-icBb950O!q+s<7c?Ci(hol5O2AZ21{o6) zfEx(A4_0MJ#?Mp!Th0n2kC*p$=!t{&cE3a@meyE8B-S!4?cDrwISSvtOwd8as;JGO zH*Me4q~nT-!4gKGS?y-nvM6Y_*UZ7z$(b^;;8F@{{ek6aVCWm4%{7y#x?|u{^Uaws zZBGfDGA#&AG0EBdzssM1DLSS%WpldZB5%o)n;SEzyK^ zB3jZ_A5+6PY8PQ~Ckc;M*O%R6Pw+d(OO%g0F~5`VsA};*y`?Fgp^58ca`WBW=jO10 z8SeUWL#TphN!Y==;gu6=OVEkcu6?hCej6x!of9z^3cnR56jk%BL#P=e`!8`SEs6Q=vZO z36rKdvaKv)GXXXtG-8TxUf7Le`r-EWhGbOI1=CETYo%!j?ZbefFdDO_MDog5-dI>` z`xPEe>(ZccjHD9=Dw)bes+!8=g(=b2kv(BiF1cRe1@GVnZPk-O#zByG)6vNFG_te= z=h|gr`23hErVaO)CN#ZoemhX}F)u{=R?<`%mgq*hkJXq!S0mq-UzUTc2O&2Jql%FQ zy*CnoEY(!Wm5uDe0$4^Ag|b#$xuKz#uibP;$W)341qWG>M9`SrNU>mg&T{1)cc}W) z%!mmH$qf<+3T421HWR7xgHfH(q{^rGF9ni0EMkk8OuWt=vn5N+tr;)Gzj%_+v5>Pf zmX3a$pw-|fHa#;fp8bF~^la-s?&3Utw0Fjce&_@55}*)#7XLnSchht(I z$-bD8dn}OVBp@JedQGLF?lDn<;(UjT%iH7P}zY;-@a*N za6ipiYTiMK*oF5MsRXC4d1w5w$v2IZ}KV8uDgRR|Oj6h;~6cVzQxRXJr@`#JCuT`Ig^7 za*)s5ZL1a^Ec)N^*<0FJ%XfWv_>GI~F+1x34xCG|y&Scsa>m*s*7g|~d0Rw8A~5Vu z10ur!dAfk{bdL?|3&L|q0G6xaZhEZ%N7o@vjJti8oX9eH-yUx zSjj$sI^keCy9!7QE~MWVj3)%-xVroph|p6g3poj968UC^L*BvXAPw-U@8C8vz>tEBc9Efoa9_LCkcnrIOm}$RB-HpDChzh zm(kjmcrg`0a1-;>%kfu`cn4&pDUt4b;p0fEjZ^7@WzLtBCtx&KeZPVoTd67 z5$&U3U!1S$*Vtb9=Y~1d>ahAEOh@&+{2bNu_2(!g)w((gN%c7$g`}f;UVDz}`B6Qu zzkB37N6vHPJR6>)dj9m)b5ORKKDtZ+yi1v7$00sM^vMMd>+jteCTrSy7Q6_jN(F#do-ixqt8>Cv! z5wJE|Htwwf>ouD$8&`ts4u8ux59 zX%X651U45yVC19S1`f2!X?v2`1$zS`By&g;M zdeSmi-~|ekwYtk$AVsy4g%BAl6z#fAwrZBEf@G>b^3;NUv`0FSBfO;_`66GG-4R92 zTnN%N%(bYy*~HVpgYoiDZJ~ON$7I3NjuYVNjwkTdpb4Pxf&%8nj{Qm(c)M%mzfqP& zKG~KOjZ)|{MSz^*SY^#WbxJC7z}m}A&6k=%Lq4wgQTJj7qtCi-U>|o zEmb5g?@7$EM64DYMZo6TgiW;w%~<#rt#nOJk8){5P&hbVFg;J0)C)-HmG-Ev4{)nq z6hZ?p+!)Il{g`)L8i!<9&V5tK_sx{CNH;sJ%|cdI-nG1mu(YW}300Uf2ot#FfrSj# zKXYlxnio2OdCLaQZ>M?abS`gQuH3MfXnWjgk%PsG<^2G&HssWR`GU*?xm09ek z4*UYV3}97Jt$u2Ms-G!bB<hOrEVY0}Lo4dPUu&^`z{d-wX-qX8&x&^( zK7IS_$peMrresPDkIgfYj=@C5t)6yD_#OFmQ!l?ZX@K@HB`fmsE7T^z0<>&s@ECVx2%oB~%w3v(t*9FaL?`G7<%zb2_4+X7qY6(qO zghsk3warML#!MzFvyI4JW4k13i}eS$JY&+_L>7;Ut=^b0&3xm&guj1n%L;E6_T>&m zzf`0n&E$foAaQ{F7bwVWX#GUb8J8rXQzdY?K4A&=LYN37R z8^ynU2(>5@#k&;nce~5L!sm49%7Q6Fp1c*qFt^<;$7bp`TGxYWe@DVl`%!Lwcf&!d zzfs+ib}e^(*dth&KWzO9<&0NgdcYsBEK&KFeCpK%*RW-dDb9|1^@%c~s S{Qm#|0RR70eo_wr + echo "Applying CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl replace -Rf /etc/crd || kubectl create -Rf /etc/crd; + echo "Done!" + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + echo "Deleting CRDs..."; + mkdir -p /etc/crd; + base64 -d /etc/config/crd-manifest.tgz.b64 | tar -xzv -C /etc/crd; + kubectl delete --ignore-not-found=true -Rf /etc/crd; + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/manifest.yaml b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/manifest.yaml new file mode 100644 index 000000000..8dc9dfb44 --- /dev/null +++ b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/manifest.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.tgz.b64: + {{- .Files.Get "files/crd-manifest.tgz" | b64enc | indent 4 }} diff --git a/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/rbac.yaml b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/rbac.yaml new file mode 100644 index 000000000..a4d498b0f --- /dev/null +++ b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/rbac.yaml @@ -0,0 +1,76 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete', 'update', 'list'] +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/validate-psp-install.yaml b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring-crd/106.0.0+up61.3.2/values.yaml b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/values.yaml new file mode 100644 index 000000000..99e63600c --- /dev/null +++ b/charts/rancher-monitoring-crd/106.0.0+up61.3.2/values.yaml @@ -0,0 +1,17 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + +image: + repository: rancher/shell + tag: v0.2.1 + +nodeSelector: {} + +tolerations: [] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/.editorconfig b/charts/rancher-monitoring/106.0.0+up61.3.2/.editorconfig new file mode 100644 index 000000000..f5ee2f461 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[files/dashboards/*.json] +indent_size = 2 +indent_style = space \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/.helmignore new file mode 100644 index 000000000..9bdbec92b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz + +unittests/ +files/dashboards/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/CHANGELOG.md b/charts/rancher-monitoring/106.0.0+up61.3.2/CHANGELOG.md new file mode 100644 index 000000000..8178169b9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8081` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/CONTRIBUTING.md b/charts/rancher-monitoring/106.0.0+up61.3.2/CONTRIBUTING.md new file mode 100644 index 000000000..f6ce2a323 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/Chart.yaml new file mode 100644 index 000000000..9d49ca31a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/Chart.yaml @@ -0,0 +1,126 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/deploys-on-os: windows + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/rancher-version: '>= 2.10.0-0 < 2.11.0-0' + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/type: cluster-tool + catalog.cattle.io/ui-component: monitoring + catalog.cattle.io/upstream-version: 57.0.3 +apiVersion: v2 +appVersion: v0.75.1 +dependencies: +- condition: grafana.enabled + name: grafana + repository: file://./charts/grafana +- condition: hardenedKubelet.enabled + name: hardenedKubelet + repository: file://./charts/hardenedKubelet +- condition: hardenedNodeExporter.enabled + name: hardenedNodeExporter + repository: file://./charts/hardenedNodeExporter +- condition: k3sServer.enabled + name: k3sServer + repository: file://./charts/k3sServer +- condition: kubeStateMetrics.enabled + name: kube-state-metrics + repository: file://./charts/kube-state-metrics +- condition: kubeAdmControllerManager.enabled + name: kubeAdmControllerManager + repository: file://./charts/kubeAdmControllerManager +- condition: kubeAdmEtcd.enabled + name: kubeAdmEtcd + repository: file://./charts/kubeAdmEtcd +- condition: kubeAdmProxy.enabled + name: kubeAdmProxy + repository: file://./charts/kubeAdmProxy +- condition: kubeAdmScheduler.enabled + name: kubeAdmScheduler + repository: file://./charts/kubeAdmScheduler +- condition: prometheus-adapter.enabled + name: prometheus-adapter + repository: file://./charts/prometheus-adapter +- condition: nodeExporter.enabled + name: prometheus-node-exporter + repository: file://./charts/prometheus-node-exporter +- condition: rke2ControllerManager.enabled + name: rke2ControllerManager + repository: file://./charts/rke2ControllerManager +- condition: rke2Etcd.enabled + name: rke2Etcd + repository: file://./charts/rke2Etcd +- condition: rke2IngressNginx.enabled + name: rke2IngressNginx + repository: file://./charts/rke2IngressNginx +- condition: rke2Proxy.enabled + name: rke2Proxy + repository: file://./charts/rke2Proxy +- condition: rke2Scheduler.enabled + name: rke2Scheduler + repository: file://./charts/rke2Scheduler +- condition: rkeControllerManager.enabled + name: rkeControllerManager + repository: file://./charts/rkeControllerManager +- condition: rkeEtcd.enabled + name: rkeEtcd + repository: file://./charts/rkeEtcd +- condition: rkeIngressNginx.enabled + name: rkeIngressNginx + repository: file://./charts/rkeIngressNginx +- condition: rkeProxy.enabled + name: rkeProxy + repository: file://./charts/rkeProxy +- condition: rkeScheduler.enabled + name: rkeScheduler + repository: file://./charts/rkeScheduler +- condition: windowsExporter.enabled + name: windowsExporter + repository: file://./charts/windowsExporter +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: file://assets/logos/rancher-monitoring.png +keywords: +- operator +- prometheus +- kube-prometheus +kubeVersion: '>=1.28.0-0' +maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: kube-prometheus-stack@sisti.pt + name: GMartinez-Sisti +- email: github@jkroepke.de + name: jkroepke +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: quentin.bisson@gmail.com + name: QuentinBisson +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +type: application +version: 106.0.0+up61.3.2 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/README.md new file mode 100644 index 000000000..46f751587 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/README.md @@ -0,0 +1,1140 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3+ + +## Get Helm Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) +- [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd alertmanagerconfigs.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd probes.monitoring.coreos.com +kubectl delete crd prometheusagents.monitoring.coreos.com +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd scrapeconfigs.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +With Helm v3, CRDs created by this chart are not updated by default and should be manually updated. +Consult also the [Helm Documentation on CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions). + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 60.x to 61.x + +This version upgrades Prometheus-Operator to v0.75.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 59.x to 60.x + +This version upgrades the Grafana chart to v8.0.x which introduces Grafana 11. This new major version of Grafana contains some breaking changes described in [Breaking changes in Grafana v11.0](https://grafana.com/docs/grafana/latest/breaking-changes/breaking-changes-v11-0/). + +### From 58.x to 59.x + +This version upgrades Prometheus-Operator to v0.74.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.74.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 57.x to 58.x + +This version upgrades Prometheus-Operator to v0.73.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 56.x to 57.x + +This version upgrades Prometheus-Operator to v0.72.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.72.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 55.x to 56.x + +This version upgrades Prometheus-Operator to v0.71.0, Prometheus to 2.49.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.71.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 54.x to 55.x + +This version upgrades Prometheus-Operator to v0.70.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.70.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 53.x to 54.x + +Grafana Helm Chart has bumped to version 7 + +Please note Grafana Helm Chart [changelog](https://github.com/grafana/helm-charts/tree/main/charts/grafana#to-700). + +### From 52.x to 53.x + +This version upgrades Prometheus-Operator to v0.69.1, Prometheus to 2.47.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.69.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 51.x to 52.x + +This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting: + +- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or +- `thanosRuler.thanosRulerSpec.objectStorageConfig` or +- `thanosRuler.thanosRulerSpec.queryConfig` or +- `prometheus.prometheusSpec.thanos.objectStorageConfig` + +you will have to need to set `existingSecret` or `secret` based on your requirement + +For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + alertmanagersConfig: + secret: + alertmanagers: + - api_version: v2 + http_config: + basic_auth: + username: some_user + password: some_pass + static_configs: + - alertmanager.thanos.io + scheme: http + timeout: 10s +``` + +or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +But it now moved to: + +```yaml +thanosRuler: + thanosRulerSpec: + objectStorageConfig: + existingSecret: + name: existing-secret-not-created-by-this-chart + key: object-storage-configs.yaml +``` + +### From 50.x to 51.x + +This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.68.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 49.x to 50.x + +This version requires Kubernetes 1.19+. + +We do not expect any breaking changes in this version. + +### From 48.x to 49.x + +This version upgrades Prometheus-Operator to v0.67.1, 0, Alertmanager to v0.26.0, Prometheus to 2.46.0 and Thanos to v0.32.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 47.x to 48.x + +This version moved all CRDs into a dedicated sub-chart. No new CRDs are introduced in this version. +See [#3548](https://github.com/prometheus-community/helm-charts/issues/3548) for more context. + +We do not expect any breaking changes in this version. + +### From 46.x to 47.x + +This version upgrades Prometheus-Operator to v0.66.0 with new CRDs (PrometheusAgent and ScrapeConfig). + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.66.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 45.x to 46.x + +This version upgrades Prometheus-Operator to v0.65.1 with new CRDs (PrometheusAgent and ScrapeConfig), Prometheus to v2.44.0 and Thanos to v0.31.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 44.x to 45.x + +This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.42.0 and Thanos to v0.30.2. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 43.x to 44.x + +This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default). + +The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead. + +### From 42.x to 43.x + +This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 41.x to 42.x + +This includes the overridability of container registry for all containers at the global level using `global.imageRegistry` or per container image. The defaults have not changed but if you were using a custom image, you will have to override the registry of said custom container image before you upgrade. + +For instance, the prometheus-config-reloader used to be configured as follow: + +```yaml + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +But it now moved to: + +```yaml + image: + registry: quay.io + repository: prometheus-operator/prometheus-config-reloader + tag: v0.60.1 + sha: "" +``` + +### From 40.x to 41.x + +This version upgrades Prometheus-Operator to v0.60.1, Prometheus to v2.39.1 and Thanos to v0.28.1. +This version also upgrades the Helm charts of kube-state-metrics to 4.20.2, prometheus-node-exporter to 4.3.0 and Grafana to 6.40.4. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.60.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +This version splits kubeScheduler recording and altering rules in separate config values. +Instead of `defaultRules.rules.kubeScheduler` the 2 new variables `defaultRules.rules.kubeSchedulerAlerting` and `defaultRules.rules.kubeSchedulerRecording` are used. + +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 38.x to 39.x + +This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 37.x to 38.x + +Reverted one of the default metrics relabelings for cAdvisor added in 36.x, due to it breaking container_network_* and various other statistics. If you do not want this change, you will need to override the `kubelet.cAdvisorMetricRelabelings`. + +### From 36.x to 37.x + +This includes some default metric relabelings for cAdvisor and apiserver metrics to reduce cardinality. If you do not want these defaults, you will need to override the `kubeApiServer.metricRelabelings` and or `kubelet.cAdvisorMetricRelabelings`. + +### From 35.x to 36.x + +This upgraded prometheus-operator to v0.57.0 and prometheus to v2.36.1 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 34.x to 35.x + +This upgraded prometheus-operator to v0.56.0 and prometheus to v2.35.0 + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.56.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 33.x to 34.x + +This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 32.x to 33.x + +This upgrades the prometheus-node-exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation. + +### From 31.x to 32.x + +This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 30.x to 31.x + +This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart. +`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may +need to be manually cleaned up after deploying the new release. + +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits prometheus-node-exporter chart recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built-in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + +### From 17.x to 18.x + +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 16.x to 17.x + +Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 15.x to 16.x + +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + +### From 14.x to 15.x + +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 13.x to 14.x + +Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 12.x to 13.x + +Version 13 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +``` + +### From 11.x to 12.x + +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + +The chart was migrated to support only helm v3 and later. + +### From 10.x to 11.x + +Version 11 upgrades prometheus-operator from 0.42.x to 0.43.x. Starting with 0.43.x an additional `AlertmanagerConfigs` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.43/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +``` + +Version 11 removes the deprecated tlsProxy via ghostunnel in favor of native TLS support the prometheus-operator gained with v0.39.0. + +### From 9.x to 10.x + +Version 10 upgrades prometheus-operator from 0.38.x to 0.42.x. Starting with 0.40.x an additional `Probes` CRD is introduced. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.42/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +``` + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeIngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `rke2IngressNginx.enabled` | Create a PushProx installation for monitoring ingress-nginx metrics in RKE2 clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repository](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizations/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. +For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: + +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. +Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. +An easy way of doing this, without compromising the default PodMonitors/ServiceMonitors discovery, is allowing Prometheus to discover all PodMonitors/ServiceMonitors within its namespace, without applying label filtering. +To do so, you can set `prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues` and `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from stable/prometheus-operator chart + +## Zero downtime + +Since `kube-prometheus-stack` is fully compatible with the `stable/prometheus-operator` chart, a migration without downtime can be achieved. +However, the old name prefix needs to be kept. If you want the new name please follow the step by step guide below (with downtime). + +You can override the name to achieve this: + +```console +helm upgrade prometheus-operator prometheus-community/kube-prometheus-stack -n monitoring --reuse-values --set nameOverride=prometheus-operator +``` + +**Note**: It is recommended to run this first with `--dry-run --debug`. + +## Redeploy with new name (downtime) + +If the **prometheus-operator** values are compatible with the new **kube-prometheus-stack** chart, please follow the below steps for migration: + +> The guide presumes that chart is deployed in `monitoring` namespace and the deployments are running there. If in other namespace, please replace the `monitoring` to the deployed namespace. + +1. Patch the PersistenceVolume created/used by the prometheus-operator chart to `Retain` claim policy: + + ```console + kubectl patch pv/ -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + ``` + + **Note:** To execute the above command, the user must have a cluster wide permission. Please refer [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +2. Uninstall the **prometheus-operator** release and delete the existing PersistentVolumeClaim, and verify PV become Released. + + ```console + helm uninstall prometheus-operator -n monitoring + kubectl delete pvc/ -n monitoring + ``` + + Additionally, you have to manually remove the remaining `prometheus-operator-kubelet` service. + + ```console + kubectl delete service/prometheus-operator-kubelet -n kube-system + ``` + + You can choose to remove all your existing CRDs (ServiceMonitors, Podmonitors, etc.) if you want to. + +3. Remove current `spec.claimRef` values to change the PV's status from Released to Available. + + ```console + kubectl patch pv/ --type json -p='[{"op": "remove", "path": "/spec/claimRef"}]' -n monitoring + ``` + +**Note:** To execute the above command, the user must have a cluster wide permission. Please refer to [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) + +After these steps, proceed to a fresh **kube-prometheus-stack** installation and make sure the current release of **kube-prometheus-stack** matching the `volumeClaimTemplate` values in the `values.yaml`. + +The binding is done via matching a specific amount of storage requested and with certain access modes. + +For example, if you had storage specified as this with **prometheus-operator**: + +```yaml +volumeClaimTemplate: + spec: + storageClassName: gp2 + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi +``` + +You have to specify matching `volumeClaimTemplate` with 50Gi storage and `ReadWriteOnce` access mode. + +Additionally, you should check the current AZ of your legacy installation's PV, and configure the fresh release to use the same AZ as the old one. If the pods are in a different AZ than the PV, the release will fail to bind the existing one, hence creating a new PV. + +This can be achieved either by specifying the labels through `values.yaml`, e.g. setting `prometheus.prometheusSpec.nodeSelector` to: + +```yaml +nodeSelector: + failure-domain.beta.kubernetes.io/zone: east-west-1a +``` + +or passing these values as `--set` overrides during installation. + +The new release should now re-attach your previously released PV with its content. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/app-README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/app-README.md new file mode 100644 index 000000000..392085438 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/app-README.md @@ -0,0 +1,46 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) / [kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.7/packages/rancher-monitoring/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Upgrading from 100.0.0+up16.6.0 to 100.1.0+up19.0.3 + +### Noticeable changes: +Grafana: +- `sidecar.dashboards.searchNamespace`, `sidecar.datasources.searchNamespace` and `sidecar.notifiers.searchNamespace` support a list of namespaces now. + +Kube-state-metrics +- the type of `collectors` is changed from Dictionary to List. +- `kubeStateMetrics.serviceMonitor.namespaceOverride` was replaced by `kube-state-metrics.namespaceOverride`. + +### Known issues: +- Occasionally, the upgrade fails with errors related to the webhook `prometheusrulemutate.monitoring.coreos.com`. This is a known issue in the upstream, and the workaround is to trigger the upgrade one more time. [32416](https://github.com/rancher/rancher/issues/32416#issuecomment-828881726) diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/.helmignore new file mode 100644 index 000000000..8cade1318 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/Chart.yaml new file mode 100644 index 000000000..7770ebbe5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/Chart.yaml @@ -0,0 +1,41 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/grafana/helm-charts + - name: Upstream Project + url: https://github.com/grafana/grafana + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-grafana +apiVersion: v2 +appVersion: 11.1.0 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.com +icon: https://artifacthub.io/image/b4fed1a7-6c8f-4945-b99d-096efa3e4116 +keywords: +- monitoring +- metric +kubeVersion: '>=1.28.0-0' +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter +- email: github@jkroepke.de + name: jkroepke +name: grafana +sources: +- https://github.com/grafana/grafana +- https://github.com/grafana/helm-charts +type: application +version: 8.3.6 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/README.md new file mode 100644 index 000000000..f758963a4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/README.md @@ -0,0 +1,778 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## Get Repo Info + +```console +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +### To 6.0.0 + +This version requires Helm >= 3.1.0. + +### To 7.0.0 + +For consistency with other Helm charts, the `global.image.registry` parameter was renamed +to `global.imageRegistry`. If you were not previously setting `global.image.registry`, no action +is required on upgrade. If you were previously setting `global.image.registry`, you will +need to instead set `global.imageRegistry`. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `podDisruptionBudget.apiVersion` | Pod disruption apiVersion | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.registry` | Image registry | `docker.io` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Overrides the Grafana image tag whose default is the chart appVersion (`Must be >= 5.0.0`) | `` | +| `image.sha` | Image sha (optional) | `` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets (can be templated) | `[]` | +| `service.enabled` | Enable grafana service | `true` | +| `service.ipFamilies` | Kubernetes service IP families | `[]` | +| `service.ipFamilyPolicy` | Kubernetes service IP family policy | `""` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations (can be templated) | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `service.externalTrafficPolicy` | change the default externalTrafficPolicy | `nil` | +| `headlessService` | Create a headless service | `false` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.pathType` | Ingress type of path | `Prefix` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.6/guide/ingress/annotations/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `ingress.ingressClassName` | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `""` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `extraLabels` | Custom labels for all manifests | `{}` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data (can be templated) | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.extraPvcLabels` | Extra labels to apply to a PVC. | `{}` | +| `persistence.subPath` | Mount a sub dir of the persistent volume (can be templated) | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | +| `persistence.disableWarning` | Hide NOTES warning, useful when persiting to a database | `false` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.registry` | init-chown-data container image registry | `docker.io` | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envFromSecrets` | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envFromConfigMaps` | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret. (passed through [tpl](https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function)) | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraVolumes` | Additional Grafana server volumes | `[]` | +| `automountServiceAccountToken` | Mounted the service account token on the grafana pod. Mandatory, if sidecars are enabled | `true` | +| `createConfigmap` | Enable creating the grafana configmap | `true` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts (values are templated) | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `global.imageRegistry` | Global image pull registry for all images. | `null` | +| `global.imagePullSecrets` | Global image pull secrets (can be templated). Allows either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). | `[]` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `lifecycleHooks` | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers) | `{}` | +| `sidecar.image.registry` | Sidecar image registry | `quay.io` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.26.0` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | +| `sidecar.alerts.enabled` | Enables the cluster wide search for alerts and adds/updates/deletes them in grafana |`false` | +| `sidecar.alerts.label` | Label that config maps with alerts should have to be added | `grafana_alert` | +| `sidecar.alerts.labelValue` | Label value that config maps with alerts should have to be added | `""` | +| `sidecar.alerts.searchNamespace` | Namespaces list. If specified, the sidecar will search for alerts config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.alerts.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.alerts.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.alerts.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/alerting/reload"` | +| `sidecar.alerts.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.alerts.initAlerts` | Set to true to deploy the alerts sidecar as an initContainer. This is needed if skipReload is true, to load any alerts defined at startup time. | `false` | +| `sidecar.alerts.extraMounts` | Additional alerts sidecar volume mounts. | `[]` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.folderUid` | Allows you to specify the static UID for the logical folder above | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.provider.foldersFromFilesStructure` | Allow Grafana to replicate dashboard structure from filesystem. | `false` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `""` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | +| `sidecar.dashboards.reloadURL` | Full url of dashboards configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/dashboards/reload"` | +| `sidecar.dashboards.skipReload` | Enabling this omits defining the REQ_USERNAME, REQ_PASSWORD, REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `""` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.datasources.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.datasources.initDatasources` | Set to true to deploy the datasource sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any datasources defined at startup time. | `false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.labelValue` | Label value that config maps with notifiers should have to be added | `""` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.notifiers.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.notifiers.reloadURL` | Full url of notifier configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/notifications/reload"` | +| `sidecar.notifiers.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | +| `sidecar.notifiers.initNotifiers` | Set to true to deploy the notifier sidecar as an initContainer in addition to a container. This is needed if skipReload is true, to load any notifiers defined at startup time. | `false` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials (can be templated). | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.automountServiceAccountToken` | Automount the service account token on all pods where is service account is used | `false` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.labels` | ServiceAccount labels | `{}` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.useExistingRole` | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `false` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `args` | Define additional args if command is used | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image.registry` | `test-framework` image registry. | `docker.io` | +| `testFramework.image.repository` | `test-framework` image repository. | `bats/bats` | +| `testFramework.image.tag` | `test-framework` image tag. | `v1.4.1` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.registry` | Curl docker image registry | `docker.io` | +| `downloadDashboardsImage.repository` | Curl docker image repository | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.73.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | RelabelConfigs to apply to samples before scraping. | `[]` | +| `serviceMonitor.metricRelabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | +| `revisionHistoryLimit` | Number of old ReplicaSets to retain | `10` | +| `imageRenderer.enabled` | Enable the image-renderer deployment & service | `false` | +| `imageRenderer.image.registry` | image-renderer Image registry | `docker.io` | +| `imageRenderer.image.repository` | image-renderer Image repository | `grafana/grafana-image-renderer` | +| `imageRenderer.image.tag` | image-renderer Image tag | `latest` | +| `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | +| `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | +| `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.envValueFrom` | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` | +| `imageRenderer.extraConfigmapMounts` | Additional image-renderer configMap volume mounts (values are templated) | `[]` | +| `imageRenderer.extraSecretMounts` | Additional image-renderer secret volume mounts | `[]` | +| `imageRenderer.extraVolumeMounts` | Additional image-renderer volume mounts | `[]` | +| `imageRenderer.extraVolumes` | Additional image-renderer volumes | `[]` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | +| `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | +| `imageRenderer.podAnnotations` | image-renderer image-renderer pod annotation | `{}` | +| `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | +| `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | +| `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | +| `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | +| `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.resources` | Set resource limits for image-renderer pods | `{}` | +| `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | +| `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | +| `imageRenderer.affinity` | Affinity settings for pod assignment | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `networkPolicy.ingress` | Enable the creation of an ingress network policy | `true` | +| `networkPolicy.egress.enabled` | Enable the creation of an egress network policy | `false` | +| `networkPolicy.egress.ports` | An array of ports to allow for the egress | `[]` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | + +### Example ingress with path + +With grafana 6.3 and above + +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts and extraVolumes + +Configure additional volumes with `extraVolumes` and volume mounts with `extraVolumeMounts`. + +Example for `extraVolumeMounts` and corresponding `extraVolumes`: + +```yaml +extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + readOnly: false + - name: dashboards + mountPath: /var/lib/grafana/dashboards + hostPath: /usr/shared/grafana/dashboards + readOnly: false + +extraVolumes: + - name: plugins + existingClaim: existing-grafana-claim + - name: dashboards + hostPath: /usr/shared/grafana/dashboards +``` + +Volumes default to `emptyDir`. Set to `persistentVolumeClaim`, +`hostPath`, `csi`, or `configMap` for other types. For a +`persistentVolumeClaim`, specify an existing claim name with +`existingClaim`. + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + loki-dashboard-quick-search: + gnetId: 12019 + revision: 2 + datasource: + - name: DS_PROMETHEUS + value: Prometheus + - name: DS_LOKI + value: Loki + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. + +Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://..svc.cluster.local/api/admin/provisioning/datasources/reload`. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example values to add a postgres datasource as a kubernetes secret: +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-datasources + labels: + grafana_datasource: 'true' # default value for: sidecar.datasources.label +stringData: + pg-db.yaml: |- + apiVersion: 1 + datasources: + - name: My pg db datasource + type: postgres + url: my-postgresql-db:5432 + user: db-readonly-user + secureJsonData: + password: 'SUperSEcretPa$$word' + jsonData: + database: my_datase + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 0 # Grafana v5.4+ + maxIdleConns: 2 # Grafana v5.4+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # allow users to edit datasources from the UI. + editable: false +``` + +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## Sidecar for alerting resources + +If the parameter `sidecar.alerts.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster (namespace defined by `sidecar.alerts.searchNamespace`) and filters out the ones with +a label as defined in `sidecar.alerts.label` (default is `grafana_alert`). The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported alerting resources are updated, however, deletions are a little more complicated (see below). + +This sidecar can be used to provision alert rules, contact points, notification policies, notification templates and mute timings as shown in [Grafana Documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/). + +To fetch the alert config which will be provisioned, use the alert provisioning API ([Grafana Documentation](https://grafana.com/docs/grafana/next/developers/http_api/alerting_provisioning/)). +You can use either JSON or YAML format. + +Example config for an alert rule: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-alert + labels: + grafana_alert: "1" +data: + k8s-alert.yml: |- + apiVersion: 1 + groups: + - orgId: 1 + name: k8s-alert + [...] +``` + +To delete provisioned alert rules is a two step process, you need to delete the configmap which defined the alert rule +and then create a configuration which deletes the alert rule. + +Example deletion configuration: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: delete-sample-grafana-alert + namespace: monitoring + labels: + grafana_alert: "1" +data: + delete-k8s-alert.yml: |- + apiVersion: 1 + deleteRules: + - orgId: 1 + uid: 16624780-6564-45dc-825c-8bded4ad92d3 +``` + +## Statically provision alerting resources +If you don't need to change alerting resources (alert rules, contact points, notification policies and notification templates) regularly you could use the `alerting` config option instead of the sidecar option above. +This will grab the alerting config and apply it statically at build time for the helm file. + +There are two methods to statically provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +alerting: + team1-alert-rules.yaml: + file: alerting/team1/rules.yaml + team2-alert-rules.yaml: + file: alerting/team2/rules.yaml + team3-alert-rules.yaml: + file: alerting/team3/rules.yaml + notification-policies.yaml: + file: alerting/shared/notification-policies.yaml + notification-templates.yaml: + file: alerting/shared/notification-templates.yaml + contactpoints.yaml: + apiVersion: 1 + contactPoints: + - orgId: 1 + name: Slack channel + receivers: + - uid: default-receiver + type: slack + settings: + # Webhook URL to be filled in + url: "" + # We need to escape double curly braces for the tpl function. + text: '{{ `{{ template "default.message" . }}` }}' + title: '{{ `{{ template "default.title" . }}` }}' +``` + +The two possibilities for static alerting resource provisioning are: + +* Inlining the file contents as shown for contact points in the above example. +* Importing a file using a relative path starting from the chart root directory as shown for the alert rules in the above example. + +### Important notes on file provisioning + +* The format of the files is defined in the [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/) on file provisioning. +* The chart supports importing YAML and JSON files. +* The filename must be unique, otherwise one volume mount will overwrite the other. +* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped. +* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance. +* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases. + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` + +## How to securely reference secrets in grafana.ini + +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. + +In grafana.ini: + +```yaml +grafana.ini: + [auth.generic_oauth] + enabled = true + client_id = $__file{/etc/secrets/auth_generic_oauth/client_id} + client_secret = $__file{/etc/secrets/auth_generic_oauth/client_secret} +``` + +Existing secret, or created along with helm: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-generic-oauth-secret +type: Opaque +stringData: + client_id: + client_secret: +``` + +Include in the `extraSecretMounts` configuration flag: + +```yaml +- extraSecretMounts: + - name: auth-generic-oauth-secret-mount + secretName: auth-generic-oauth-secret + defaultMode: 0440 + mountPath: /etc/secrets/auth_generic_oauth + readOnly: true +``` + +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + +## Image Renderer Plug-In + +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) + +```yaml +imageRenderer: + enabled: true +``` + +### Image Renderer NetworkPolicy + +By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + ha_listen_address: ${POD_IP}:9094 + ha_advertise_address: ${POD_IP}:9094 + + alerting: + enabled: false +``` diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/dashboards/custom-dashboard.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/NOTES.txt new file mode 100644 index 000000000..a40f666a4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/NOTES.txt @@ -0,0 +1,55 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo + + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: + {{- range .Values.ingress.hosts }} + http://{{ . }} + {{- end }} +{{- else }} + Get the Grafana URL to visit by running these commands in the same shell: + {{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ include "grafana.namespace" . }} -w {{ include "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} + {{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "grafana.namespace" . }} -l "app.kubernetes.io/name={{ include "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ include "grafana.namespace" . }} port-forward $POD_NAME 3000 + {{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if and (not .Values.persistence.enabled) (not .Values.persistence.disableWarning) }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_config.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_config.tpl new file mode 100644 index 000000000..b866217f2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_config.tpl @@ -0,0 +1,172 @@ +{{/* + Generate config map data + */}} +{{- define "grafana.configData" -}} +{{ include "grafana.assertNoLeakedSecrets" . }} +{{- $files := .Files }} +{{- $root := . -}} +{{- with .Values.plugins }} +plugins: {{ join "," . }} +{{- end }} +grafana.ini: | +{{- range $elem, $elemVal := index .Values "grafana.ini" }} + {{- if not (kindIs "map" $elemVal) }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} +{{- end }} +{{- range $key, $value := index .Values "grafana.ini" }} + {{- if kindIs "map" $value }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{- if kindIs "invalid" $elemVal }} + {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} + {{- else }} + {{ $elem }} = {{ $elemVal }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} + +{{- range $key, $value := .Values.datasources }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.notifiers }} +{{- if not (hasKey $value "secret") }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.alerting }} +{{- if (hasKey $value "file") }} +{{ $key }}: +{{- toYaml ( $files.Get $value.file ) | nindent 2 }} +{{- else if (or (hasKey $value "secret") (hasKey $value "secretFile"))}} +{{/* will be stored inside secret generated by "configSecret.yaml"*/}} +{{- else }} +{{ $key }}: | + {{- tpl (toYaml $value | nindent 2) $root }} +{{- end }} +{{- end }} + +{{- range $key, $value := .Values.dashboardProviders }} +{{ $key }}: | + {{- toYaml $value | nindent 2 }} +{{- end }} + +{{- if .Values.dashboards }} +download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} +{{ $dashboardProviders := .Values.dashboardProviders }} +{{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + {{- if not $value.acceptHeader }} + -H "Accept: application/json" \ + {{- else }} + -H "Accept: {{ $value.acceptHeader }}" \ + {{- end }} + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.basic }} + -H "Authorization: Basic {{ $value.basic }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} + -H "Content-Type: application/json;charset=UTF-8" \ + {{- end }} + {{- $dpPath := "" -}} + {{- range $kd := (index $dashboardProviders "dashboardproviders.yaml").providers }} + {{- if eq $kd.name $provider }} + {{- $dpPath = $kd.options.path }} + {{- end }} + {{- end }} + {{- if $value.url }} + "{{ $value.url }}" \ + {{- else }} + "https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download" \ + {{- end }} + {{- if $value.datasource }} + {{- if kindIs "string" $value.datasource }} + | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g' \ + {{- end }} + {{- if kindIs "slice" $value.datasource }} + {{- range $value.datasource }} + | sed '/-- .* --/! s/${{"{"}}{{ .name }}}/{{ .value }}/g' \ + {{- end }} + {{- end }} + {{- end }} + {{- if $value.b64content }} + | base64 -d \ + {{- end }} + > "{{- if $dpPath -}}{{ $dpPath }}{{- else -}}/var/lib/grafana/dashboards/{{ $provider }}{{- end -}}/{{ $key }}.json" + {{ end }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* + Generate dashboard json config map data + */}} +{{- define "grafana.configDashboardProviderData" -}} +provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + folderUid: '{{ .Values.sidecar.dashboards.provider.folderUid }}' + {{- end }} + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} + options: + foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end -}} + +{{- define "grafana.secretsData" -}} +{{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} +admin-user: {{ .Values.adminUser | b64enc | quote }} +{{- if .Values.adminPassword }} +admin-password: {{ .Values.adminPassword | b64enc | quote }} +{{- else }} +admin-password: {{ include "grafana.password" . }} +{{- end }} +{{- end }} +{{- if not .Values.ldap.existingSecret }} +ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_helpers.tpl new file mode 100644 index 000000000..68d2d815d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,305 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create }} +{{- default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else }} +{{- default "default" .Values.serviceAccount.nameTest }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.extraLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "grafana.imageRenderer.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.imageRenderer.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels ImageRenderer +*/}} +{{- define "grafana.imageRenderer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) }} +{{- if $secret }} +{{- index $secret "data" "admin-password" }} +{{- else }} +{{- (randAlphaNum 40) | b64enc | quote }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "grafana.rbac.apiVersion" -}} +{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" }} +{{- else }} +{{- print "rbac.authorization.k8s.io/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} +{{- if and ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) }} +{{- print "networking.k8s.io/v1" }} +{{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- print "networking.k8s.io/v1beta1" }} +{{- else }} +{{- print "extensions/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "grafana.hpa.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "autoscaling/v2" }} +{{- print "autoscaling/v2" }} +{{- else }} +{{- print "autoscaling/v2beta2" }} +{{- end }} +{{- end }} + +{{/* +Return the appropriate apiVersion for podDisruptionBudget. +*/}} +{{- define "grafana.podDisruptionBudget.apiVersion" -}} +{{- if $.Values.podDisruptionBudget.apiVersion }} +{{- print $.Values.podDisruptionBudget.apiVersion }} +{{- else if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} +{{- print "policy/v1" }} +{{- else }} +{{- print "policy/v1beta1" }} +{{- end }} +{{- end }} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} +{{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" }} +{{- end }} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} +{{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "root" . "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "grafana.imagePullSecrets" -}} +{{- $root := .root }} +{{- range (concat .root.Values.global.imagePullSecrets .imagePullSecrets) }} +{{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml (dict "name" (tpl .name $root)) | trim }} +{{- else }} +- name: {{ tpl . $root }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* + Checks whether or not the configSecret secret has to be created + */}} +{{- define "grafana.shouldCreateConfigSecret" -}} +{{- $secretFound := false -}} +{{- range $key, $value := .Values.datasources }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} + {{- if hasKey $value "secret" }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} + {{- if (or (hasKey $value "secret") (hasKey $value "secretFile")) }} + {{- $secretFound = true}} + {{- end }} +{{- end }} +{{- $secretFound}} +{{- end -}} + +{{/* + Checks whether the user is attempting to store secrets in plaintext + in the grafana.ini configmap +*/}} +{{/* grafana.assertNoLeakedSecrets checks for sensitive keys in values */}} +{{- define "grafana.assertNoLeakedSecrets" -}} + {{- $sensitiveKeysYaml := ` +sensitiveKeys: +- path: ["database", "password"] +- path: ["smtp", "password"] +- path: ["security", "secret_key"] +- path: ["security", "admin_password"] +- path: ["auth.basic", "password"] +- path: ["auth.ldap", "bind_password"] +- path: ["auth.google", "client_secret"] +- path: ["auth.github", "client_secret"] +- path: ["auth.gitlab", "client_secret"] +- path: ["auth.generic_oauth", "client_secret"] +- path: ["auth.okta", "client_secret"] +- path: ["auth.azuread", "client_secret"] +- path: ["auth.grafana_com", "client_secret"] +- path: ["auth.grafananet", "client_secret"] +- path: ["azure", "user_identity_client_secret"] +- path: ["unified_alerting", "ha_redis_password"] +- path: ["metrics", "basic_auth_password"] +- path: ["external_image_storage.s3", "secret_key"] +- path: ["external_image_storage.webdav", "password"] +- path: ["external_image_storage.azure_blob", "account_key"] +` | fromYaml -}} + {{- if $.Values.assertNoLeakedSecrets -}} + {{- $grafanaIni := index .Values "grafana.ini" -}} + {{- range $_, $secret := $sensitiveKeysYaml.sensitiveKeys -}} + {{- $currentMap := $grafanaIni -}} + {{- $shouldContinue := true -}} + {{- range $index, $elem := $secret.path -}} + {{- if and $shouldContinue (hasKey $currentMap $elem) -}} + {{- if eq (len $secret.path) (add1 $index) -}} + {{- if not (regexMatch "\\$(?:__(?:env|file|vault))?{[^}]+}" (index $currentMap $elem)) -}} + {{- fail (printf "Sensitive key '%s' should not be defined explicitly in values. Use variable expansion instead. You can disable this client-side validation by changing the value of assertNoLeakedSecrets." (join "." $secret.path)) -}} + {{- end -}} + {{- else -}} + {{- $currentMap = index $currentMap $elem -}} + {{- end -}} + {{- else -}} + {{- $shouldContinue = false -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_pod.tpl new file mode 100644 index 000000000..44526e523 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/_pod.tpl @@ -0,0 +1,1306 @@ +{{- define "grafana.pod" -}} +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- $root := . -}} +{{- with .Values.schedulerName }} +schedulerName: "{{ . }}" +{{- end }} +serviceAccountName: {{ include "grafana.serviceAccountName" . }} +automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} +{{- with .Values.securityContext }} +securityContext: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.dnsPolicy }} +dnsPolicy: {{ .Values.dnsPolicy }} +{{- end }} +{{- with .Values.dnsConfig }} +dnsConfig: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.priorityClassName }} +priorityClassName: {{ . }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.extraInitContainers (and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts) (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources) (and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers)) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- $registry := include "system_default_registry" . | default .Values.initChownData.image.registry -}} + {{- if .Values.initChownData.image.sha }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + {{- with .Values.initChownData.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + command: + - chown + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }} + - /var/lib/grafana + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- $registry := include "system_default_registry" . | default .Values.downloadDashboardsImage.registry -}} + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + env: + {{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.downloadDashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.downloadDashboards.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl . $root }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.alerts.enabled .Values.sidecar.alerts.initAlerts }} + - name: {{ include "grafana.name" . }}-init-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end }} +{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }} + - name: {{ include "grafana.name" . }}-init-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: "LIST" + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (.Values.sidecar.datasources.searchNamespace | join ",") . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end }} +{{- if and .Values.sidecar.notifiers.enabled .Values.sidecar.notifiers.initNotifiers }} + - name: {{ include "grafana.name" . }}-init-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- with .Values.extraInitContainers }} + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 2 }} +{{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} +enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} +containers: +{{- if and .Values.sidecar.alerts.enabled (not .Values.sidecar.alerts.initAlerts) }} + - name: {{ include "grafana.name" . }}-sc-alerts + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.alerts.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.alerts.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.alerts.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.alerts.label }}" + {{- with .Values.sidecar.alerts.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.alerts.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/alerting" + - name: RESOURCE + value: {{ quote .Values.sidecar.alerts.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.alerts.searchNamespace }} + - name: NAMESPACE + value: {{ . | join "," | quote }} + {{- end }} + {{- with .Values.sidecar.alerts.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: {{ quote . }} + {{- end }} + {{- with .Values.sidecar.alerts.script }} + - name: SCRIPT + value: {{ quote . }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.alerts.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.alerts.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.alerts.watchServerTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchServerTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.alerts.watchClientTimeout }} + {{- if ne .Values.sidecar.alerts.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.alerts.watchClientTimeout with .Values.sidecar.alerts.watchMethod %s" .Values.sidecar.alerts.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.alerts.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- with .Values.sidecar.alerts.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ include "grafana.name" . }}-sc-dashboard + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.dashboards.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + {{- with .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.dashboards.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.folderAnnotation }} + - name: FOLDER_ANNOTATION + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.dashboards.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- if not .Values.sidecar.dashboards.skipReload }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + - name: REQ_URL + value: {{ .Values.sidecar.dashboards.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: {{ .Values.sidecar.dashboards.watchClientTimeout | quote }} + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- with .Values.sidecar.dashboards.extraMounts }} + {{- toYaml . | trim | nindent 6 }} + {{- end }} +{{- end}} +{{- if and .Values.sidecar.datasources.enabled (not .Values.sidecar.datasources.initDatasources) }} + - name: {{ include "grafana.name" . }}-sc-datasources + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- range $key, $value := .Values.sidecar.datasources.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- with .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ include "grafana.name" . }}-sc-notifiers + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.notifiers.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + {{- with .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote . }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: {{ quote .Values.sidecar.notifiers.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- with .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if .Values.sidecar.notifiers.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.notifiers.script }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.notifiers.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.notifiers.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.notifiers.watchServerTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchServerTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.notifiers.watchClientTimeout }} + {{- if ne .Values.sidecar.notifiers.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.notifiers.watchClientTimeout with .Values.sidecar.notifiers.watchMethod %s" .Values.sidecar.notifiers.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.notifiers.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.sidecar.plugins.enabled }} + - name: {{ include "grafana.name" . }}-sc-plugins + {{- $registry := include "system_default_registry" . | default .Values.sidecar.image.registry -}} + {{- if .Values.sidecar.image.sha }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} + - name: METHOD + value: {{ .Values.sidecar.plugins.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.plugins.label }}" + {{- if .Values.sidecar.plugins.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.plugins.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/plugins" + - name: RESOURCE + value: {{ quote .Values.sidecar.plugins.resource }} + {{- with .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.plugins.searchNamespace }} + - name: NAMESPACE + value: "{{ tpl (. | join ",") $root }}" + {{- end }} + {{- with .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ . }}" + {{- end }} + {{- with .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ . }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.plugins.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.plugins.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" +{{- end}} + - name: {{ .Chart.Name }} + {{- $registry := include "system_default_registry" . | default .Values.image.registry -}} + {{- if .Values.image.sha }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.args }} + args: + {{- range .Values.args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" + {{- with .Values.persistence.subPath }} + subPath: {{ tpl . $root }} + {{- end }} + {{- with .Values.dashboards }} + {{- range $provider, $dashboards := . }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardsConfigMaps }} + {{- range (keys . | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} + {{- end }} + {{- with .Values.datasources }} + {{- $datasources := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $datasources .) "secret")) }} {{/*check if current datasource should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.notifiers }} + {{- $notifiers := . }} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $notifiers .) "secret")) }} {{/*check if current notifier should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.alerting }} + {{- $alertingmap := .}} + {{- range (keys . | sortAlpha) }} + {{- if (or (hasKey (index $.Values.alerting .) "secret") (hasKey (index $.Values.alerting .) "secretFile")) }} {{/*check if current alerting entry should be handeled as secret */}} + - name: config-secret + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- else }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.dashboardProviders }} + {{- range (keys . | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/{{ . }}" + subPath: {{ . | quote }} + {{- end }} + {{- end }} + {{- with .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + mountPath: "/etc/grafana/provisioning/alerting" + {{- end}} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml + {{- end}} + {{- end}} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + mountPath: "/etc/grafana/provisioning/plugins" + {{- end}} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" + {{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.podPortName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.gossipPortName }}-tcp + containerPort: 9094 + protocol: TCP + - name: {{ .Values.gossipPortName }}-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ (tpl .Values.admin.existingSecret .) | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ include "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- if .Values.imageRenderer.enabled }} + - name: GF_RENDERING_SERVER_URL + value: http://{{ include "grafana.fullname" . }}-image-renderer.{{ include "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render + - name: GF_RENDERING_CALLBACK_URL + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ include "grafana.fullname" . }}.{{ include "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 10 }} + {{- end }} + {{- range $key, $value := .Values.env }} + - name: "{{ tpl $key $ }}" + value: "{{ tpl (print $value) $ }}" + {{- end }} + {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }} + envFrom: + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + - secretRef: + name: {{ include "grafana.fullname" . }}-env + {{- end }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- range .Values.envFromConfigMaps }} + - configMapRef: + name: {{ tpl .name $ }} + optional: {{ .optional | default false }} + {{- if .prefix }} + prefix: {{ tpl .prefix $ }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.lifecycleHooks }} + lifecycle: + {{- tpl (toYaml .) $root | nindent 6 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- with .Values.extraContainers }} + {{- tpl . $ | nindent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- with .Values.nodeSelector }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- tpl (toYaml .) $root | nindent 2 }} +{{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- with .Values.tolerations }} + {{- toYaml . | nindent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ include "grafana.fullname" . }} + {{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} + {{- if and .Values.createConfigmap $createConfigSecret }} + - name: config-secret + secret: + secretName: {{ include "grafana.fullname" . }}-config-secret + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ include "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ tpl (.Values.persistence.existingClaim | default (include "grafana.fullname" .)) . }} + {{- else if and .Values.persistence.enabled (has .Values.persistence.type $sts) }} + {{/* nothing */}} + {{- else }} + - name: storage + {{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory + {{- with .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.alerts.enabled }} + - name: sc-alerts-volume + emptyDir: + {{- with .Values.sidecar.alerts.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: + {{- with .Values.sidecar.dashboards.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ include "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- end }} + {{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: + {{- with .Values.sidecar.datasources.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.plugins.enabled }} + - name: sc-plugins-volume + emptyDir: + {{- with .Values.sidecar.plugins.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: + {{- with .Values.sidecar.notifiers.sizeLimit }} + sizeLimit: {{ . }} + {{- else }} + {} + {{- end }} + {{- end }} + {{- range .Values.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 6 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- end }} + {{- end }} + {{- range .Values.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 6 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 6 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 6 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 6 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} + {{- end }} + {{- with .Values.extraContainerVolumes }} + {{- tpl (toYaml .) $root | nindent 2 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrole.yaml new file mode 100644 index 000000000..3af4b62b6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingClusterRole) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} +rules: + {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end}} + {{- with .Values.rbac.extraClusterRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..bda9431a2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +roleRef: + kind: ClusterRole + {{- if .Values.rbac.useExistingClusterRole }} + name: {{ .Values.rbac.useExistingClusterRole }} + {{- else }} + name: {{ include "grafana.fullname" . }}-clusterrole + {{- end }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configSecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configSecret.yaml new file mode 100644 index 000000000..55574b9bb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configSecret.yaml @@ -0,0 +1,43 @@ +{{- $createConfigSecret := eq (include "grafana.shouldCreateConfigSecret" .) "true" -}} +{{- if and .Values.createConfigmap $createConfigSecret }} +{{- $files := .Files }} +{{- $root := . -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ include "grafana.fullname" . }}-config-secret" + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- range $key, $value := .Values.alerting }} + {{- if (hasKey $value "secretFile") }} + {{- $key | nindent 2 }}: + {{- toYaml ( $files.Get $value.secretFile ) | b64enc | nindent 4}} + {{/* as of https://helm.sh/docs/chart_template_guide/accessing_files/ this will only work if you fork this chart and add files to it*/}} + {{- end }} +{{- end }} +stringData: +{{- range $key, $value := .Values.datasources }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.notifiers }} +{{- if (hasKey $value "secret") }} +{{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} +{{- end }} +{{- end }} +{{- range $key, $value := .Values.alerting }} +{{ if (hasKey $value "secret") }} + {{- $key | nindent 2 }}: | + {{- tpl (toYaml $value.secret | nindent 4) $root }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100644 index 000000000..b412c4d1f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.sidecar.dashboards.enabled .Values.sidecar.dashboards.SCProvider }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "grafana.fullname" . }}-config-dashboards + namespace: {{ include "grafana.namespace" . }} +data: + {{- include "grafana.configDashboardProviderData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap.yaml new file mode 100644 index 000000000..0a2edf47e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.createConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- if or .Values.configMapAnnotations .Values.annotations }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.configMapAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +data: + {{- include "grafana.configData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100644 index 000000000..b96ce7202 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,38 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ include "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.enabled }} + {{ $.Values.sidecar.dashboards.label }}: {{ $.Values.sidecar.dashboards.labelValue | quote }} + {{- end }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} + {{- print $key | nindent 2 }}.json: + {{- if hasKey $value "json" }} + |- + {{- $value.json | nindent 6 }} + {{- end }} + {{- if hasKey $value "file" }} + {{- toYaml ( $files.Get $value.file ) | nindent 4}} + {{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/deployment.yaml new file mode 100644 index 000000000..46c016faa --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/deployment.yaml @@ -0,0 +1,53 @@ +{{- if (and (not .Values.useStatefulSet) (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc"))) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include "grafana.configData" . | sha256sum }} + {{- if .Values.dashboards }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + {{- end }} + checksum/sc-dashboard-provider-config: {{ include "grafana.configDashboardProviderData" . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include "grafana.secretsData" . | sha256sum }} + {{- end }} + {{- if .Values.envRenderSecret }} + checksum/secret-env: {{ tpl (toYaml .Values.envRenderSecret) . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/extra-manifests.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 000000000..a9bb3b6ba --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/headless-service.yaml new file mode 100644 index 000000000..3028589d3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,22 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-headless + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: {{ .Values.gossipPortName }}-tcp + port: 9094 +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/hpa.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/hpa.yaml new file mode 100644 index 000000000..46bbcb49a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if .Values.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }} + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + {{- if has .Values.persistence.type $sts }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ include "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.autoscaling.behavior }} + behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-deployment.yaml new file mode 100644 index 000000000..acf03e4da --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-deployment.yaml @@ -0,0 +1,199 @@ +{{ if .Values.imageRenderer.enabled }} +{{- $root := . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }} + replicas: {{ .Values.imageRenderer.replicas }} + {{- end }} + revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + + {{- with .Values.imageRenderer.deploymentStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 8 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.imageRenderer.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imageRenderer.schedulerName }} + schedulerName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ . }}" + {{- end }} + {{- with .Values.imageRenderer.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.image.pullSecrets }} + imagePullSecrets: + {{- range . }} + - name: {{ tpl . $root }} + {{- end}} + {{- end }} + containers: + - name: {{ .Chart.Name }}-image-renderer + {{- $registry := include "system_default_registry" | default .Values.imageRenderer.image.registry -}} + {{- if .Values.imageRenderer.image.sha }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}@sha256:{{ .Values.imageRenderer.image.sha }}" + {{- else }} + image: "{{ $registry }}{{ .Values.imageRenderer.image.repository }}:{{ .Values.imageRenderer.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.imageRenderer.image.pullPolicy }} + {{- if .Values.imageRenderer.command }} + command: + {{- range .Values.imageRenderer.command }} + - {{ . }} + {{- end }} + {{- end}} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} + env: + - name: HTTP_PORT + value: {{ .Values.imageRenderer.service.targetPort | quote }} + {{- if .Values.imageRenderer.serviceMonitor.enabled }} + - name: ENABLE_METRICS + value: "true" + {{- end }} + {{- range $key, $value := .Values.imageRenderer.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: + {{- tpl (toYaml $value) $ | nindent 16 }} + {{- end }} + {{- range $key, $value := .Values.imageRenderer.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- with .Values.imageRenderer.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /tmp + name: image-renderer-tmpfs + {{- range .Values.imageRenderer.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + mountPath: {{ tpl .mountPath $root }} + subPath: {{ tpl (.subPath | default "") $root }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.imageRenderer.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.imageRenderer.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.imageRenderer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.imageRenderer.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} + {{- with .Values.imageRenderer.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: image-renderer-tmpfs + emptyDir: {} + {{- range .Values.imageRenderer.extraConfigmapMounts }} + - name: {{ tpl .name $root }} + configMap: + name: {{ tpl .configMap $root }} + {{- with .items }} + items: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- end }} + {{- range .Values.imageRenderer.extraSecretMounts }} + {{- if .secretName }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- with .items }} + items: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- else if .projected }} + - name: {{ .name }} + projected: + {{- toYaml .projected | nindent 12 }} + {{- else if .csi }} + - name: {{ .name }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- end }} + {{- end }} + {{- range .Values.imageRenderer.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-hpa.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-hpa.yaml new file mode 100644 index 000000000..b0f0059b7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-hpa.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }} +apiVersion: {{ include "grafana.hpa.apiVersion" . }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer + helm.sh/chart: {{ include "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "grafana.fullname" . }}-image-renderer + minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }} + maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }} + metrics: + {{- if .Values.imageRenderer.autoscaling.targetMemory }} + - type: Resource + resource: + name: memory + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.targetCPU }} + - type: Resource + resource: + name: cpu + {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }} + targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- else }} + target: + type: Utilization + averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }} + {{- end }} + {{- end }} + {{- if .Values.imageRenderer.autoscaling.behavior }} + behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-network-policy.yaml new file mode 100644 index 000000000..bcbd24976 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-network-policy.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitIngress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-ingress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer ingress traffic from grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Ingress + ingress: + - ports: + - port: {{ .Values.imageRenderer.service.targetPort }} + protocol: TCP + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}} + {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} + +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer-egress + namespace: {{ include "grafana.namespace" . }} + annotations: + comment: Limit image-renderer egress traffic to grafana +spec: + podSelector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + {{- with .Values.imageRenderer.podLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + policyTypes: + - Egress + egress: + # allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # talk only to grafana + - ports: + - port: {{ .Values.service.targetPort }} + protocol: TCP + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "grafana.namespace" . }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 14 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 14 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-service.yaml new file mode 100644 index 000000000..f8da127cf --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imageRenderer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + {{- with .Values.imageRenderer.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - name: {{ .Values.imageRenderer.service.portName }} + port: {{ .Values.imageRenderer.service.port }} + protocol: TCP + targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- with .Values.imageRenderer.appProtocol }} + appProtocol: {{ . }} + {{- end }} + selector: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-servicemonitor.yaml new file mode 100644 index 000000000..5d9f09d26 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/image-renderer-servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.imageRenderer.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }}-image-renderer + {{- if .Values.imageRenderer.serviceMonitor.namespace }} + namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.imageRenderer.labels" . | nindent 4 }} + {{- with .Values.imageRenderer.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.imageRenderer.service.portName }} + {{- with .Values.imageRenderer.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.imageRenderer.serviceMonitor.path }} + scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }} + {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.imageRenderer.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}-image-renderer" + selector: + matchLabels: + {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.imageRenderer.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/ingress.yaml new file mode 100644 index 000000000..b2ffd8109 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/ingress.yaml @@ -0,0 +1,78 @@ +{{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} + {{- with .Values.ingress.tls }} + tls: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ tpl . $ | quote }} + http: + paths: + {{- with $extraPaths }} + {{- toYaml . | nindent 10 }} + {{- end }} + - path: {{ $ingressPath }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + {{- if $ingressApiIsStable }} + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- with $ingressPath }} + path: {{ . }} + {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/networkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 000000000..4cd3ed697 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + policyTypes: + {{- if .Values.networkPolicy.ingress }} + - Ingress + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + - Egress + {{- end }} + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + + {{- if .Values.networkPolicy.egress.enabled }} + egress: + {{- if not .Values.networkPolicy.egress.blockDNSResolution }} + - ports: + - port: 53 + protocol: UDP + {{- end }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.ingress }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "grafana.fullname" . }}-client: "true" + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/nginx-config.yaml new file mode 100644 index 000000000..557471f6f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $http_host; + proxy_pass http://localhost:3000; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_once off; + + {{- if eq .Values.global.cattle.clusterId "local" -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- else -}} + sub_filter '"appSubUrl":""' '"appSubUrl":"/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "grafana.namespace" . }}/services/http:{{ template "grafana.fullname" . }}:{{ .Values.service.port }}/proxy"'; + {{- end -}} + + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..05251214a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: {{ include "grafana.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ . }} + {{- end }} + {{- with .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100644 index 000000000..ebbdef5b5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, with DAC_OVERRIDE and CHOWN + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'csi' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/pvc.yaml new file mode 100644 index 000000000..b5fd5d4a3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/pvc.yaml @@ -0,0 +1,41 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.extraPvcLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if and (.Values.persistence.lookupVolumeName) (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)) }} + volumeName: {{ (lookup "v1" "PersistentVolumeClaim" (include "grafana.namespace" .) (include "grafana.fullname" .)).spec.volumeName }} + {{- end }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/role.yaml new file mode 100644 index 000000000..4b5edd978 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.rbac.pspEnabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }} +rules: + {{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}] + {{- end }} + {{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }} + - apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] + {{- end }} + {{- with .Values.rbac.extraRoleRules }} + {{- toYaml . | nindent 2 }} + {{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/rolebinding.yaml new file mode 100644 index 000000000..58f77c6b0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + {{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} + {{- else }} + name: {{ include "grafana.fullname" . }} + {{- end }} +subjects: +- kind: ServiceAccount + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret-env.yaml new file mode 100644 index 000000000..eb14aac70 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }}-env + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ tpl ($val | toString) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret.yaml new file mode 100644 index 000000000..fd2ca50f4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- include "grafana.secretsData" . | nindent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/service.yaml new file mode 100644 index 000000000..022328c11 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/service.yaml @@ -0,0 +1,67 @@ +{{- if .Values.service.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- with .Values.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: LoadBalancer + {{- with .Values.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerClass }} + loadBalancerClass: {{ . }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- if .Values.service.ipFamilyPolicy }} + ipFamilyPolicy: {{ .Values.service.ipFamilyPolicy }} + {{- end }} + {{- if .Values.service.ipFamilies }} + ipFamilies: {{ .Values.service.ipFamilies | toYaml | nindent 2 }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- with .Values.service.appProtocol }} + appProtocol: {{ . }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- with .Values.extraExposePorts }} + {{- tpl (toYaml . | nindent 4) $root }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/serviceaccount.yaml new file mode 100644 index 000000000..ffca0717a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.autoMount | default .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + name: {{ include "grafana.serviceAccountName" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/servicemonitor.yaml new file mode 100644 index 000000000..160a9c556 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,66 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} + {{- else }} + namespace: {{ include "grafana.namespace" . }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + endpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + honorLabels: true + path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- with .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 6 }} + {{- end }} + metricRelabelings: + {{- if .Values.serviceMonitor.metricRelabelings }} + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ include "grafana.namespace" . }} + {{- with .Values.serviceMonitor.targetLabels }} + targetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/statefulset.yaml new file mode 100644 index 000000000..49278083e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,58 @@ +{{- $sts := list "sts" "StatefulSet" "statefulset" -}} +{{- if (or (.Values.useStatefulSet) (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (has .Values.persistence.type $sts)))}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "grafana.fullname" . }} + namespace: {{ include "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ include "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + {{- if .Values.persistence.enabled}} + volumeClaimTemplates: + - metadata: + name: storage + spec: +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" .Values.persistence.accessModes }} +{{- $_ := required "Must provide at least one access mode for persistent volumes used by Grafana" (first .Values.persistence.accessModes) }} + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ required "Must provide size for persistent volumes used by Grafana" .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-configmap.yaml new file mode 100644 index 000000000..01c96c924 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ include "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100644 index 000000000..1821772a4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "grafana.fullname" . }}-test + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - csi + - secret +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-role.yaml new file mode 100644 index 000000000..cb4c78204 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,17 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100644 index 000000000..f40d791f6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "grafana.fullname" . }}-test + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "grafana.fullname" . }}-test +subjects: + - kind: ServiceAccount + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100644 index 000000000..38fba3596 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ include "grafana.serviceAccountNameTest" . }} + namespace: {{ include "grafana.namespace" . }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test.yaml new file mode 100644 index 000000000..83aaa185c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,53 @@ +{{- if .Values.testFramework.enabled }} +{{- $root := . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: {{ include "grafana.namespace" . }} +spec: + serviceAccountName: {{ include "grafana.serviceAccountNameTest" . }} + {{- with .Values.testFramework.securityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if or .Values.image.pullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "grafana.imagePullSecrets" (dict "root" $root "imagePullSecrets" .Values.image.pullSecrets) | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ template "system_default_registry" . | default .Values.testFramework.image.registry }}/{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + {{- with .Values.testFramework.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tests + configMap: + name: {{ include "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/values.yaml new file mode 100644 index 000000000..c338ae041 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/grafana/values.yaml @@ -0,0 +1,1365 @@ +global: + cattle: + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # Can be templated. + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + +rbac: + create: true + ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true) + # useExistingRole: name-of-some-role + # useExistingClusterRole: name-of-some-clusterRole + pspEnabled: false + pspUseAppArmor: false + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: + ## ServiceAccount labels. + labels: {} + ## Service account annotations. Can be templated. + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789000:role/iam-role-name-here + + ## autoMount is deprecated in favor of automountServiceAccountToken + # autoMount: false + automountServiceAccountToken: true + +replicas: 1 + +## Create a headless service for the deployment +headlessService: false + +## Should the service account be auto mounted on the pod +automountServiceAccountToken: true + +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# apiVersion: "" +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + # Overrides the Grafana image tag whose default is the chart appVersion + tag: 11.1.0 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Can be templated. + ## + pullSecrets: [] + # - myRegistrKeySecretName + +testFramework: + enabled: false + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# dns configuration for pod +dnsPolicy: ~ +dnsConfig: {} + # nameservers: + # - 8.8.8.8 + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + +containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# Enable creating the grafana configmap +createConfigmap: true + +# Extra configmaps to mount in grafana pods +# Values are templated. +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +# Apply extra labels to common labels. +extraLabels: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 8.9.1 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + envFromSecret: "" + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## Pod Annotations +# podAnnotations: {} + +## ConfigMap Annotations +# configMapAnnotations: {} + # argocd.argoproj.io/sync-options: Replace=true + +## Pod Labels +# podLabels: {} + +podPortName: grafana +gossipPortName: gossip +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + enabled: true + type: ClusterIP + # Set the ip family policy to configure dual-stack see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services) + ipFamilyPolicy: "" + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. Can be IPv4 and/or IPv6. + ipFamilies: [] + loadBalancerIP: "" + loadBalancerClass: "" + loadBalancerSourceRanges: [] + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + +serviceMonitor: + ## If true, a ServiceMonitor CR is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + metricRelabelings: [] + targetLabels: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + + # pathType is only for k8s >= 1.1= + pathType: Prefix + + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + ## Or for k8s > 1.19 + # - path: /* + # pathType: Prefix + # backend: + # service: + # name: ssl-redirect + # port: + # name: use-annotation + + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + +## Additional init containers (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: "" +# extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # selectorLabels: {} + ## Sub-directory of the PV to mount. Can be templated. + # subPath: "" + ## Name of an existing PVC. Can be templated. + # existingClaim: + ## Extra labels to apply to a PVC. + extraPvcLabels: {} + disableWarning: false + + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + + ## If 'lookupVolumeName' is set to true, Helm will attempt to retrieve + ## the current value of 'spec.volumeName' and incorporate it into the template. + lookupVolumeName: true + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the grafana-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + securityContext: + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + capabilities: + add: + - CHOWN + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + ## Name of the secret. Can be templated. + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Optionally define args if command is used +## Needed if using `hashicorp/envconsul` to manage secrets +## By default no arguments are set +# args: +# - "-secret" +# - "secret/grafana" +# - "./grafana" + +## Extra environment variables that will be pass onto deployment pods +## +## to provide grafana with access to CloudWatch on AWS EKS: +## 1. create an iam role of type "Web identity" with provider oidc.eks.* (note the provider for later) +## 2. edit the "Trust relationships" of the role, add a line inside the StringEquals clause using the +## same oidc eks provider as noted before (same as the existing line) +## also, replace NAMESPACE and prometheus-operator-grafana with the service account namespace and name +## +## "oidc.eks.us-east-1.amazonaws.com/id/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:sub": "system:serviceaccount:NAMESPACE:prometheus-operator-grafana", +## +## 3. attach a policy to the role, you can use a built in policy called CloudWatchReadOnlyAccess +## 4. use the following env: (replace 123456789000 and iam-role-name-here with your aws account number and role name) +## +## env: +## AWS_ROLE_ARN: arn:aws:iam::123456789000:role/iam-role-name-here +## AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token +## AWS_REGION: us-east-1 +## +## 5. uncomment the EKS section in extraSecretMounts: below +## 6. uncomment the annotation section in the serviceAccount: above +## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn + +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc. +## If the secret values contains "{{", they'll need to be properly escaped so that they are not interpreted by Helm +## ref: https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function +envRenderSecret: {} + +## The names of secrets in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key. +## Name is templated. +envFromSecrets: [] +## - name: secret-name +## prefix: prefix +## optional: true + +## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment +## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key. +## Name is templated. +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core +envFromConfigMaps: [] +## - name: configmap-name +## prefix: prefix +## optional: true + +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + # + # for AWS EKS (cloudwatch) use the following (see also instruction in env: above) + # - name: aws-iam-token + # mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + # readOnly: true + # projected: + # defaultMode: 420 + # sources: + # - serviceAccountToken: + # audience: sts.amazonaws.com + # expirationSeconds: 86400 + # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume-0 + # mountPath: /mnt/volume0 + # readOnly: true + # - name: extra-volume-1 + # mountPath: /mnt/volume1 + # readOnly: true + # - name: grafana-secrets + # mountPath: /mnt/volume2 + +## Additional Grafana server volumes +extraVolumes: [] + # - name: extra-volume-0 + # existingClaim: volume-claim + # - name: extra-volume-1 + # hostPath: + # path: /usr/shared/ + # type: "" + # - name: grafana-secrets + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" + +## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request +lifecycleHooks: {} + # postStart: + # exec: + # command: [] + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + ## You can also use other plugin download URL, as long as they are valid zip files, + ## and specify the name of the plugin after the semicolon. Like this: + # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true +# - name: CloudWatch +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 +# deleteDatasources: [] +# - name: Prometheus + +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} + # rules.yaml: + # apiVersion: 1 + # groups: + # - orgId: 1 + # name: '{{ .Chart.Name }}_my_rule_group' + # folder: my_first_folder + # interval: 60s + # rules: + # - uid: my_id_1 + # title: my_first_rule + # condition: A + # data: + # - refId: A + # datasourceUid: '-100' + # model: + # conditions: + # - evaluator: + # params: + # - 3 + # type: gt + # operator: + # type: and + # query: + # params: + # - A + # reducer: + # type: last + # type: query + # datasource: + # type: __expr__ + # uid: '-100' + # expression: 1==0 + # intervalMs: 1000 + # maxDataPoints: 43200 + # refId: A + # type: math + # dashboardUid: my_dashboard + # panelId: 123 + # noDataState: Alerting + # for: 60s + # annotations: + # some_key: some_value + # labels: + # team: sre_team_1 + # contactpoints.yaml: + # secret: + # apiVersion: 1 + # contactPoints: + # - orgId: 1 + # name: cp_1 + # receivers: + # - uid: first_uid + # type: pagerduty + # settings: + # integrationKey: XXX + # severity: critical + # class: ping failure + # component: Grafana + # group: app-stack + # summary: | + # {{ `{{ include "default.message" . }}` }} + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # token: '' + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # token: '' + # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' + # local-dashboard-azure: + # url: https://example.com/repository/test-azure.json + # basic: '' + # acceptHeader: '*/*' + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/ + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ else }}''{{ end }}" +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://api.github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 1.27.4 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + readinessProbe: {} + livenessProbe: {} + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO + alerts: + enabled: false + # Additional environment variables for the alerts sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with alert are marked with + label: grafana_alert + # value of label that the configmaps with alert are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for alert config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/alerting/reload" + # Absolute path to shell script to execute after a alert got reloaded + script: null + skipReload: false + # This is needed if skipReload is true, to load any alerts defined at startup time. + # Deploy the alert sidecar as an initContainer. + initAlerts: false + # Additional alert sidecar volume mounts + extraMounts: [] + # Sets the size limit of the alert sidecar emptyDir volume + sizeLimit: {} + dashboards: + enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces. + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # If specified, the sidecar will look for annotation with this name to create folder and put graph here. + # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. + folderAnnotation: null + # Endpoint to send request to reload alerts + reloadURL: "http://localhost:3000/api/admin/provisioning/dashboards/reload" + # Absolute path to shell script to execute after a configmap got reloaded + script: null + skipReload: false + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # folder UID. will be automatically generated if not specified + folderUid: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + # allow Grafana to replicate dashboard structure from filesystem + foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] + # Sets the size limit of the dashboard sidecar emptyDir volume + sizeLimit: {} + datasources: + enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with datasources are marked with + label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null + skipReload: true + # This is needed if skipReload is true, to load any datasources defined at startup time. + # Deploy the datasources sidecar as an initContainer. + initDatasources: true + # Sets the size limit of the datasource sidecar emptyDir volume + sizeLimit: {} + plugins: + enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with plugins are marked with + label: grafana_plugin + # value of label that the configmaps with plugins are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for plugin config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload plugins + reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null + skipReload: false + # Deploy the datasource sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any plugins defined at startup time. + initPlugins: false + # Sets the size limit of the plugin sidecar emptyDir volume + sizeLimit: {} + notifiers: + enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH + # search in configmap, secret or both + resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # + # Endpoint to send request to reload notifiers + reloadURL: "http://localhost:3000/api/admin/provisioning/notifications/reload" + # Absolute path to shell script to execute after a notifier got reloaded + script: null + skipReload: false + # Deploy the notifier sidecar as an initContainer in addition to a container. + # This is needed if skipReload is true, to load any notifiers defined at startup time. + initNotifiers: false + # Sets the size limit of the notifier sidecar emptyDir volume + sizeLimit: {} + +## Override the deployment namespace +## +namespaceOverride: "" + +## Number of old ReplicaSets to retain +## +revisionHistoryLimit: 10 + +## Add a seperate remote image renderer deployment/service +imageRenderer: + deploymentStrategy: {} + # Enable the image-renderer deployment & service + enabled: false + replicas: 1 + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPU: "60" + targetMemory: "" + behavior: {} + image: + # image-renderer Image repository + repository: rancher/mirrored-grafana-grafana-image-renderer + # image-renderer Image tag + tag: 3.11.1 + # image-renderer Image sha (optional) + sha: "" + # image-renderer ImagePullPolicy + pullPolicy: Always + # extra environment variables + env: + HTTP_HOST: "0.0.0.0" + # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 + # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true + + ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. + ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core + ## Renders in container spec as: + ## env: + ## ... + ## - name: + ## valueFrom: + ## + envValueFrom: {} + # ENV_NAME: + # configMapKeyRef: + # name: configmap-name + # key: value_key + + # image-renderer deployment serviceAccount + serviceAccountName: "" + # image-renderer deployment securityContext + securityContext: {} + # image-renderer deployment container securityContext + containerSecurityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ['ALL'] + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ## image-renderer pod annotation + podAnnotations: {} + # image-renderer deployment Host Aliases + hostAliases: [] + # image-renderer deployment priority class + priorityClassName: '' + service: + # Enable the image-renderer service + enabled: true + # image-renderer service port name + portName: 'http' + # image-renderer service port used by both service and deployment + port: 8081 + targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" + serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + relabelings: [] + # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels + targetLabels: [] + # - targetLabel1 + # - targetLabel2 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http + # In case a sub_path is used this needs to be added to the image renderer callback + grafanaSubPath: "" + # name of the image-renderer port on the pod + podPortName: http + # number of image-renderer replica sets to keep + revisionHistoryLimit: 10 + networkPolicy: + # Enable a NetworkPolicy to limit inbound traffic to only the created grafana pods + limitIngress: true + # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods + limitEgress: false + # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled) + extraIngressSelectors: [] + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + ## Node labels for pod assignment + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + # + nodeSelector: {} + + ## Tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## Affinity for pod assignment (evaluated as template) + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: "default-scheduler" + + # Extra configmaps to mount in image-renderer pods + extraConfigmapMounts: [] + + # Extra secrets to mount in image-renderer pods + extraSecretMounts: [] + + # Extra volumes to mount in image-renderer pods + extraVolumeMounts: [] + + # Extra volumes for image-renderer pods + extraVolumes: [] + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + ingress: true + ## @param networkPolicy.ingress When true enables the creation + ## an ingress network policy + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + ## + ## + ## + ## + ## + ## + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing grafana to connect to external data sources from kubernetes cluster. + enabled: false + ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ports: [] + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} + ## + ## + ## + ## + ## + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false +useStatefulSet: false +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword + +# assertNoLeakedSecrets is a helper function defined in _helpers.tpl that checks if secret +# values are not exposed in the rendered grafana.ini configmap. It is enabled by default. +# +# To pass values into grafana.ini without exposing them in a configmap, use variable expansion: +# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#variable-expansion +# +# Alternatively, if you wish to allow secret values to be exposed in the rendered grafana.ini configmap, +# you can disable this check by setting assertNoLeakedSecrets to false. +assertNoLeakedSecrets: true diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/Chart.yaml new file mode 100644 index 000000000..6ba7e5cdc --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: hardenedKubelet +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedKubelet/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/Chart.yaml new file mode 100644 index 000000000..a27b69ad6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: hardenedNodeExporter +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/hardenedNodeExporter/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/Chart.yaml new file mode 100644 index 000000000..7cc1e8f2d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: k3sServer +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/k3sServer/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/Chart.yaml new file mode 100644 index 000000000..1360e832b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,32 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-kube-state-metrics +apiVersion: v2 +appVersion: 2.12.0 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +- email: david@0xdc.me + name: dotdc +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +type: application +version: 5.21.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/README.md new file mode 100644 index 000000000..843be89e6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/README.md @@ -0,0 +1,85 @@ +# kube-state-metrics Helm Chart + +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics + +You can upgrade in-place: + +1. [get repository info](#get-repository-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repository + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: + +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-state-metrics +``` + +### kube-rbac-proxy + +You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry). +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-read +rules: + - apiGroups: [ "" ] + resources: ["services/kube-state-metrics"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/NOTES.txt new file mode 100644 index 000000000..3589c24ec --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,23 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints are now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "kube-state-metrics.fullname" . }}"] + verbs: + - get +``` +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100644 index 000000000..ed277fbb5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,196 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +{{- if .Values.selectorOverride }} +{{ toYaml .Values.selectorOverride }} +{{- else }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "kube-state-metrics.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +The image to use for kube-state-metrics +*/}} +{{- define "kube-state-metrics.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +The image to use for kubeRBACProxy +*/}} +{{- define "kubeRBACProxy.image" -}} +{{- $registry := (include "monitoring_registry" .) }} +{{- if .Values.kubeRBACProxy.image.sha }} +{{- if $registry }} +{{- printf "%s/%s:%s@%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }} +{{- end }} +{{- else }} +{{- if $registry }} +{{- printf "%s/%s:%s" $registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 000000000..025cd47a8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + endpointSelector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + egress: + {{- if and .Values.networkPolicy.cilium .Values.networkPolicy.cilium.kubeApiServerSelector }} + {{ toYaml .Values.networkPolicy.cilium.kubeApiServerSelector | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + - port: {{ .Values.service.port | quote }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + - port: {{ .Values.selfMonitor.telemetryPort | default 8081 | quote }} + protocol: TCP + {{ end }} +{{ end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..cf9f628d0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/crs-configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/crs-configmap.yaml new file mode 100644 index 000000000..d38a75a51 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/crs-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.customResourceState.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config.yaml: | + {{- toYaml .Values.customResourceState.config | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/deployment.yaml new file mode 100644 index 000000000..2c63048cc --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,313 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: +{{ toYaml .Values.annotations | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + replicas: {{ .Values.replicas }} + {{- if not .Values.autosharding.enabled }} + strategy: + type: {{ .Values.updateStrategy | default "RollingUpdate" }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] + {{- end }} + template: + metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + {{- $servicePort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - name: {{ template "kube-state-metrics.name" . }} + {{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + args: + {{- if .Values.extraArgs }} + {{- .Values.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --port={{ $servicePort }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- $namespaces := list }} + {{- if .Values.namespaces }} + {{- range $ns := join "," .Values.namespaces | split "," }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + {{- if .Values.releaseNamespace }} + {{- $namespaces = append $namespaces ( include "kube-state-metrics.namespace" . ) }} + {{- end }} + {{- if $namespaces }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + {{- if .Values.namespacesDenylist }} + - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + {{- end }} + {{- if .Values.kubeconfig.enabled }} + - --kubeconfig=/opt/k8s/.kube/config + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - --telemetry-host=127.0.0.1 + - --telemetry-port={{ $telemetryPort }} + {{- else }} + {{- if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} + - --telemetry-port={{ $telemetryPort }} + {{- end }} + {{- end }} + {{- if .Values.customResourceState.enabled }} + - --custom-resource-state-config-file=/etc/customresourcestate/config.yaml + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumeMounts) }} + volumeMounts: + {{- if .Values.kubeconfig.enabled }} + - name: kubeconfig + mountPath: /opt/k8s/.kube/ + readOnly: true + {{- end }} + {{- if .Values.customResourceState.enabled }} + - name: customresourcestate-config + mountPath: /etc/customresourcestate + readOnly: true + {{- end }} + {{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: {{ include "kube-state-metrics.image" . }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ $telemetryPort }} + name: "metrics" + {{- end }} + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: /healthz + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.hostNetwork }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-http + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port | default 8080}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port=8888 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.service.port | default 8080}} + name: "http" + - containerPort: 8888 + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8888 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- if .Values.selfMonitor.enabled }} + - name: kube-rbac-proxy-telemetry + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }} + {{- end }} + - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }} + - --upstream=http://127.0.0.1:{{ $telemetryPort }}/ + - --proxy-endpoints-port=8889 + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + {{- with .Values.kubeRBACProxy.volumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + image: {{ include "kubeRBACProxy.image" . }} + ports: + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" + - containerPort: 8889 + name: "metrics-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: 8889 + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: +{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }} +{{- end }} +{{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: +{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }} +{{- end }} + {{- end }} + {{- end }} + {{- with .Values.containers }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.topologySpreadConstraints | indent 8 }} + {{- end }} + {{- if or (.Values.kubeconfig.enabled) (.Values.customResourceState.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }} + volumes: + {{- if .Values.kubeconfig.enabled}} + - name: kubeconfig + secret: + secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + {{- end }} + {{- if .Values.kubeRBACProxy.enabled}} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + {{- end }} + {{- if .Values.customResourceState.enabled}} + - name: customresourcestate-config + configMap: + name: {{ template "kube-state-metrics.fullname" . }}-customresourcestate-config + {{- end }} + {{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/extra-manifests.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/extra-manifests.yaml new file mode 100644 index 000000000..567f7bf32 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml new file mode 100644 index 000000000..6af008450 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubeconfig.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +type: Opaque +data: + config: '{{ .Values.kubeconfig.secret }}' +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/networkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/networkpolicy.yaml new file mode 100644 index 000000000..309b38ec5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.networkPolicy.enabled (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +spec: + {{- if .Values.networkPolicy.egress }} + ## Deny all egress by default + egress: + {{- toYaml .Values.networkPolicy.egress | nindent 4 }} + {{- end }} + ingress: + {{- if .Values.networkPolicy.ingress }} + {{- toYaml .Values.networkPolicy.ingress | nindent 4 }} + {{- else }} + ## Allow ingress on default ports by default + - ports: + - port: {{ .Values.service.port | default 8080 }} + protocol: TCP + {{- if .Values.selfMonitor.enabled }} + {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}} + - port: {{ $telemetryPort }} + protocol: TCP + {{- end }} + {{- end }} + podSelector: + {{- if .Values.networkPolicy.podSelector }} + {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/pdb.yaml new file mode 100644 index 000000000..3771b511d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget -}} +{{ if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} +apiVersion: policy/v1 +{{- else -}} +apiVersion: policy/v1beta1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100644 index 000000000..8905e113e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.podSecurityPolicy.enabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100644 index 000000000..654e4a3d5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.podSecurityPolicy.enabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100644 index 000000000..5b62a18bd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.podSecurityPolicy.enabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rbac-configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rbac-configmap.yaml new file mode 100644 index 000000000..671dc9d66 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rbac-configmap.yaml @@ -0,0 +1,22 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-state-metrics.fullname" . }}-rbac-config + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- if .Values.annotations }} + annotations: + {{ toYaml .Values.annotations | nindent 4 }} + {{- end }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "kube-state-metrics.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/role.yaml new file mode 100644 index 000000000..017087837 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/role.yaml @@ -0,0 +1,215 @@ +{{- if not (kindIs "slice" .Values.collectors) }} +{{- fail "Collectors need to be a List since kube-state-metrics chart 3.2.2. Please check README for more information."}} +{{- end }} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (ternary (join "," .Values.namespaces | split "," ) (list "") (eq $.Values.rbac.useClusterRole false)) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq $.Values.rbac.useClusterRole false }} +kind: Role +{{- else }} +kind: ClusterRole +{{- end }} +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq $.Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} +rules: +{{ if has "certificatesigningrequests" $.Values.collectors }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if has "configmaps" $.Values.collectors }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if has "cronjobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "daemonsets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "deployments" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpoints" $.Values.collectors }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if has "endpointslices" $.Values.collectors }} +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: ["list", "watch"] +{{ end -}} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "ingresses" $.Values.collectors }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "jobs" $.Values.collectors }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if has "leases" $.Values.collectors }} +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +{{ end -}} +{{ if has "limitranges" $.Values.collectors }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "namespaces" $.Values.collectors }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if has "networkpolicies" $.Values.collectors }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if has "nodes" $.Values.collectors }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumeclaims" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if has "persistentvolumes" $.Values.collectors }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if has "poddisruptionbudgets" $.Values.collectors }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "pods" $.Values.collectors }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicasets" $.Values.collectors }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "replicationcontrollers" $.Values.collectors }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if has "resourcequotas" $.Values.collectors }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if has "secrets" $.Values.collectors }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "services" $.Values.collectors }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if has "statefulsets" $.Values.collectors }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if has "storageclasses" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if has "volumeattachments" $.Values.collectors }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{- if $.Values.kubeRBACProxy.enabled }} +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] +{{- end }} +{{- if $.Values.customResourceState.enabled }} +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: ["list", "watch"] +{{- end }} +{{ if $.Values.rbac.extraRules }} +{{ toYaml $.Values.rbac.extraRules }} +{{ end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 000000000..330651b73 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,24 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (join "," $.Values.namespaces) | split "," }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "kube-state-metrics.labels" $ | indent 4 }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/service.yaml new file mode 100644 index 000000000..90c235148 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + {{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port | default 8080}} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port | default 8080}} + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.selfMonitor.telemetryNodePort }} + nodePort: {{ .Values.selfMonitor.telemetryNodePort }} + {{- end }} + {{ end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if .Values.autosharding.enabled }} + clusterIP: None +{{- else if .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + selector: + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100644 index 000000000..c302bc7ca --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +metadata: + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100644 index 000000000..577981b08 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,126 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} + {{- with .Values.prometheus.monitor.annotations }} + annotations: + {{- tpl (toYaml . | nindent 4) $ }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- with .Values.prometheus.monitor.targetLabels }} + targetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }} + {{- if .Values.prometheus.monitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- with .Values.prometheus.monitor.namespaceSelector }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} + endpoints: + - port: http + {{- if or .Values.prometheus.monitor.http.interval .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.http.interval | default .Values.prometheus.monitor.interval }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.scrapeTimeout .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.http.scrapeTimeout | default .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.proxyUrl .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.http.proxyUrl | default .Values.prometheus.monitor.proxyUrl }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.enableHttp2 .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.http.enableHttp2 | default .Values.prometheus.monitor.enableHttp2 }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.honorLabels .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.metricRelabelings .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml (.Values.prometheus.monitor.http.metricRelabelings | default .Values.prometheus.monitor.metricRelabelings) | nindent 8 }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.relabelings .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml (.Values.prometheus.monitor.http.relabelings | default .Values.prometheus.monitor.relabelings) | nindent 8 }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.scheme .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.http.scheme | default .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.tlsConfig .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml (.Values.prometheus.monitor.http.tlsConfig | default .Values.prometheus.monitor.tlsConfig) | nindent 8 }} + {{- end }} + {{- if or .Values.prometheus.monitor.http.bearerTokenFile .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.http.bearerTokenFile | default .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with (.Values.prometheus.monitor.http.bearerTokenSecret | default .Values.prometheus.monitor.bearerTokenSecret) }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} + - port: metrics + {{- if or .Values.prometheus.monitor.metrics.interval .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.metrics.interval | default .Values.prometheus.monitor.interval }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.scrapeTimeout .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.metrics.scrapeTimeout | default .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.proxyUrl .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.metrics.proxyUrl | default .Values.prometheus.monitor.proxyUrl }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.enableHttp2 .Values.prometheus.monitor.enableHttp2 }} + enableHttp2: {{ .Values.prometheus.monitor.metrics.enableHttp2 | default .Values.prometheus.monitor.enableHttp2 }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.honorLabels .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.relabelings .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml (.Values.prometheus.monitor.metrics.relabelings | default .Values.prometheus.monitor.relabelings) | nindent 8 }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.scheme .Values.prometheus.monitor.scheme }} + scheme: {{ .Values.prometheus.monitor.metrics.scheme | default .Values.prometheus.monitor.scheme }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.tlsConfig .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml (.Values.prometheus.monitor.metrics.tlsConfig | default .Values.prometheus.monitor.tlsConfig) | nindent 8 }} + {{- end }} + {{- if or .Values.prometheus.monitor.metrics.bearerTokenFile .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.monitor.metrics.bearerTokenFile | default .Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- with (.Values.prometheus.monitor.metrics.bearerTokenSecret | default .Values.prometheus.monitor.bearerTokenSecret) }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100644 index 000000000..489de147c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,26 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100644 index 000000000..73b37a4f6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.serviceAccountName" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml new file mode 100644 index 000000000..f46305b51 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/templates/verticalpodautoscaler.yaml @@ -0,0 +1,44 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + {{- include "kube-state-metrics.labels" . | indent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-state-metrics.name" . }} + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{ toYaml .Values.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{ toYaml .Values.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + {{- if .Values.autosharding.enabled }} + kind: StatefulSet + {{- else }} + kind: Deployment + {{- end }} + name: {{ template "kube-state-metrics.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/values.yaml new file mode 100644 index 000000000..ab111f809 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kube-state-metrics/values.yaml @@ -0,0 +1,523 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + registry: docker.io + repository: rancher/mirrored-kube-state-metrics-kube-state-metrics + tag: v2.12.0 + sha: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +global: + cattle: + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +# Change the deployment strategy when autosharding is disabled. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +# The default is "RollingUpdate" as per Kubernetes defaults. +# During a release, 'RollingUpdate' can lead to two running instances for a short period of time while 'Recreate' can create a small gap in data. +# updateStrategy: Recreate + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + +# If false then the user will opt out of automounting API credentials. +automountServiceAccountToken: true + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + nodePort: 0 + loadBalancerIP: "" + # Only allow access to the loadBalancerIP from these IPs + loadBalancerSourceRanges: [] + clusterIP: "" + annotations: {} + +## Additional labels to add to all resources +customLabels: {} + # app: kube-state-metrics + +## Override selector labels +selectorOverride: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + + # Add permissions for CustomResources' apiGroups in Role/ClusterRole. Should be used in conjunction with Custom Resource State Metrics configuration + # Example: + # - apiGroups: ["monitoring.coreos.com"] + # resources: ["prometheuses"] + # verbs: ["list", "watch"] + extraRules: [] + +# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled). +# The requests are served through the same service but requests are then HTTPS. +kubeRBACProxy: + enabled: false + image: + repository: rancher/mirrored-brancz-kube-rbac-proxy + tag: v0.18.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-prxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + + ## volumeMounts enables mounting custom volumes in rbac-proxy containers + ## Useful for TLS certificates and keys + volumeMounts: [] + # - mountPath: /etc/tls + # name: kube-rbac-proxy-tls + # readOnly: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + # If false then the user will opt out of automounting API credentials. + automountServiceAccountToken: true + +prometheus: + monitor: + enabled: false + annotations: {} + additionalLabels: {} + namespace: "" + namespaceSelector: [] + jobLabel: "" + targetLabels: [] + podTargetLabels: [] + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + selectorOverride: {} + + ## kube-state-metrics endpoint + http: + interval: "" + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + enableHttp2: false + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + + ## selfMonitor endpoint + metrics: + interval: "" + scrapeTimeout: "" + proxyUrl: "" + ## Whether to enable HTTP2 for servicemonitor + enableHttp2: false + honorLabels: false + metricRelabelings: [] + relabelings: [] + scheme: "" + ## File to read bearer token for scraping targets + bearerTokenFile: "" + ## Secret to mount to read bearer token for scraping targets. The secret needs + ## to be in the same namespace as the service monitor and accessible by the + ## Prometheus Operator + bearerTokenSecret: {} + # name: secret-name + # key: key-name + tlsConfig: {} + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + enabled: false + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +## Configure network policy for kube-state-metrics +networkPolicy: + enabled: false + # networkPolicy.flavor -- Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + ## Configure the cilium network policy kube-apiserver selector + # cilium: + # kubeApiServerSelector: + # - toEntities: + # - kube-apiserver + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app.kubernetes.io/name: kube-state-metrics + +securityContext: + enabled: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Topology spread constraints for pod assignment +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +topologySpreadConstraints: [] + +# Annotations to be added to the deployment/statefulset +annotations: {} + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. +collectors: + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - leases + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - volumeattachments + +# Enabling kubeconfig will pass the --kubeconfig argument to the container +kubeconfig: + enabled: false + # base64 encoded kube-config file + secret: + +# Enabling support for customResourceState, will create a configMap including your config that will be read from kube-state-metrics +customResourceState: + enabled: false + # Add (Cluster)Role permissions to list/watch the customResources defined in the config to rbac.extraRules + config: {} + +# Enable only the release namespace for collecting resources. By default all namespaces are collected. +# If releaseNamespace and namespaces are both set a merged list will be collected. +releaseNamespace: false + +# Comma-separated list(string) or yaml list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" + +# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, +# only namespaces that are excluded in namespaces-denylist will be used. +namespacesDenylist: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overridden +# If you set telemetryNodePort, you must also set service.type to NodePort +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 + # telemetryNodePort: 0 + +# Enable vertical pod autoscaler support for kube-state-metrics +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: [] + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# volumeMounts are used to add custom volume mounts to deployment. +# See example below +volumeMounts: [] +# - mountPath: /etc/config +# name: config-volume + +# volumes are used to add custom volumes to deployment +# See example below +volumes: [] +# - configMap: +# name: cm-for-volume +# name: config-volume + +# Extra manifests to deploy as an array +extraManifests: [] + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: prometheus-extra + # data: + # extra-data: "value" + +## Containers allows injecting additional containers. +containers: [] + # - name: crd-init + # image: kiwigrid/k8s-sidecar:latest + +## InitContainers allows injecting additional initContainers. +initContainers: [] + # - name: crd-sidecar + # image: kiwigrid/k8s-sidecar:latest + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/Chart.yaml new file mode 100644 index 000000000..69663b1b1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: kubeAdmControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/Chart.yaml new file mode 100644 index 000000000..e24cd5c2b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: kubeAdmEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/Chart.yaml new file mode 100644 index 000000000..b968cb8fe --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: kubeAdmProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/Chart.yaml new file mode 100644 index 000000000..df91f2e3e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: kubeAdmScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 000000000..bed274d82 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,27 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-prometheus-adapter +apiVersion: v1 +appVersion: v0.11.2 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/kubernetes-sigs/prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/kubernetes-sigs/prometheus-adapter +version: 4.10.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/README.md new file mode 100644 index 000000000..d77bb0c92 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/README.md @@ -0,0 +1,160 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/kubernetes-sigs/prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Helm Repositories Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Helm Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Helm Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Helm Chart + +```console +helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### To 4.2.0 + +Readiness and liveness probes are now fully configurable through values `readinessProbe` and `livenessProbe`. The previous values have been kept as defaults. + +### To 4.0.0 + +Previously, security context of the container was set directly in the deployment template. This release makes it configurable through the new configuration variable `securityContext` whilst keeping the previously set values as defaults. Furthermore, previous variable `runAsUser` is now set in `securityContext` and is not used any longer. Please, use `securityContext.runAsUser` instead. In the same security context, `seccompProfile` has been enabled and set to type `RuntimeDefault`. + +### To 3.0.0 + +Due to a change in deployment labels, the upgrade requires `helm upgrade --force` in order to re-create the deployment. + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: | + sum by (<<.GroupBy>>) ( + rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + ) + nodeQuery: | + sum by (<<.GroupBy>>) ( + avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + - + avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + ) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 000000000..b7b9b9932 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 000000000..edbb829b2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,113 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "k8s-prometheus-adapter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "k8s-prometheus-adapter.labels" }} +helm.sh/chart: {{ include "k8s-prometheus-adapter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "k8s-prometheus-adapter.name" . }} +{{- include "k8s-prometheus-adapter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "k8s-prometheus-adapter.selectorLabels" }} +app.kubernetes.io/name: {{ include "k8s-prometheus-adapter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "k8s-prometheus-adapter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- print "policy/v1" -}} +{{- else -}} + {{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/certmanager.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/certmanager.yaml new file mode 100644 index 000000000..4e32c964c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/certmanager.yaml @@ -0,0 +1,76 @@ +{{- if .Values.certManager.enabled -}} +--- +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert + duration: {{ .Values.certManager.caCertDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.prometheus-adapter" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + ca: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }}-root-cert +--- +# Finally, generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-cert + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + duration: {{ .Values.certManager.certDuration }} + issuerRef: + name: {{ template "k8s-prometheus-adapter.fullname" . }}-root-issuer + dnsNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }} + - {{ template "k8s-prometheus-adapter.fullname" . }}.{{ include "k8s-prometheus-adapter.namespace" . }}.svc +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml new file mode 100644 index 000000000..6701e6ba0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-delegator.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-reader.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-reader.yaml new file mode 100644 index 000000000..fe1304884 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-auth-reader.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rbac.useAuthReaderClusterRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml new file mode 100644 index 000000000..67efd2aa2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-binding-resource-reader.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml new file mode 100644 index 000000000..2c690a03c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/cluster-role-resource-reader.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/configmap.yaml new file mode 100644 index 000000000..17f415d97 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/configmap.yaml @@ -0,0 +1,97 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 000000000..8b7b4e555 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 000000000..0cc692083 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,24 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 000000000..f441e1bdb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: {{ toYaml .Values.rbac.customMetrics.resources | nindent 2 }} + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/deployment.yaml new file mode 100644 index 000000000..03267b5e3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/deployment.yaml @@ -0,0 +1,151 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + {{- if or .Values.customAnnotations .Values.deploymentAnnotations }} + annotations: + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.deploymentAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: + replicas: {{ .Values.replicas }} + strategy: {{ toYaml .Values.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} + template: + metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.customAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + {{- if .Values.dnsPolicy }} + dnsPolicy: {{ .Values.dnsPolicy }} + {{- end}} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --prometheus-url={{ tpl .Values.prometheus.url . }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + {{- if .Values.extraArguments }} + {{- toYaml .Values.extraArguments | trim | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.listenPort }} + name: https + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + {{- with .Values.extraContainers }} + {{- toYaml . | nindent 6 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if or .Values.tls.enable .Values.certManager.enabled }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 000000000..21339af12 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.external }} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml new file mode 100644 index 000000000..05547bd32 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role-binding-hpa.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 000000000..71783fd4b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: {{ toYaml .Values.rbac.externalMetrics.resources | nindent 2 }} + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/pdb.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/pdb.yaml new file mode 100644 index 000000000..205761a9f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/pdb.yaml @@ -0,0 +1,23 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: {{ include "k8s-prometheus-adapter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/psp.yaml new file mode 100644 index 000000000..ec26af502 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/psp.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.psp.create (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- with (merge .Values.customAnnotations .Values.psp.annotations) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + hostPorts: + - min: {{ .Values.listenPort }} + max: {{ .Values.listenPort }} + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 000000000..0cc9fff6a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,34 @@ +{{- if .Values.rules.resource}} +{{- if .Capabilities.APIVersions.Has "apiregistration.k8s.io/v1" }} +apiVersion: apiregistration.k8s.io/v1 +{{- else }} +apiVersion: apiregistration.k8s.io/v1beta1 +{{- end }} +kind: APIService +metadata: +{{- if or .Values.certManager.enabled .Values.customAnnotations }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" (include "k8s-prometheus-adapter.namespace" .) (include "k8s-prometheus-adapter.fullname" .) | quote }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} + {{- if .Values.tls.enable }} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + {{- if not (or .Values.tls.enable .Values.certManager.enabled) }} + insecureSkipTLSVerify: true + {{- end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 000000000..3c247e48d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 000000000..73d895304 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml new file mode 100644 index 000000000..f01997e62 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/role-binding-auth-reader.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create (not .Values.rbac.useAuthReaderClusterRole) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 000000000..3e7e8887b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + {{- if .Values.customAnnotations }} + annotations: + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/service.yaml new file mode 100644 index 000000000..4879385eb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + {{- if or .Values.service.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} + {{- end }} + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} + ports: + - port: {{ .Values.service.port }} + name: https + protocol: TCP + targetPort: https + selector: + {{- include "k8s-prometheus-adapter.selectorLabels" . | indent 4 }} + type: {{ .Values.service.type }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/serviceaccount.yaml new file mode 100644 index 000000000..30a169ae0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "k8s-prometheus-adapter.labels" . | indent 4 }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ include "k8s-prometheus-adapter.namespace" . }} +{{- if or .Values.serviceAccount.annotations .Values.customAnnotations }} + annotations: + {{- if .Values.serviceAccount.annotations }} + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} + {{- if .Values.customAnnotations }} + {{- toYaml .Values.customAnnotations | nindent 4 }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/values.yaml new file mode 100644 index 000000000..e3eeb388a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-adapter/values.yaml @@ -0,0 +1,292 @@ +affinity: {} + +topologySpreadConstraints: [] + +image: + repository: rancher/mirrored-prometheus-adapter-prometheus-adapter + # if not set appVersion field from Chart.yaml is used + tag: v0.12.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +## Override the release namespace (for multi-namespace deployments in combined charts) +namespaceOverride: "" + +## Additional annotations to add to all resources +customAnnotations: {} + # role: custom-metrics + +## Additional labels to add to all resources +customLabels: {} + # monitoring: prometheus-adapter + +# Url to access prometheus +prometheus: + # Value is templated + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +# k8s 1.21 needs fsGroup to be set for non root deployments +# ref: https://github.com/kubernetes/kubernetes/issues/70679 +podSecurityContext: + fsGroup: 10001 + +# SecurityContext of the container +# ref. https://kubernetes.io/docs/tasks/configure-pod-container/security-context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + +rbac: + # Specifies whether RBAC resources should be created + create: true + # Specifies if a Cluster Role should be used for the Auth Reader + useAuthReaderClusterRole: false + externalMetrics: + resources: ["*"] + customMetrics: + resources: ["*"] + +psp: + # Specifies whether PSP resources should be created + create: false + # Annotations added to the pod security policy + annotations: {} + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +# Custom DNS configuration to be added to prometheus-adapter pods +dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +# Configure liveness probe +# https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#Probe +livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure readiness probe +readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 5 + +# Configure startup probe +# Use if prometheus-adapter takes a long time to finish startup e.g. polling a lot of API versions in cluster +startupProbe: {} + +rules: + default: true + + custom: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_custom_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + + external: [] + # - seriesQuery: '{__name__=~"^some_metric_count$"}' + # resources: + # template: <<.Resource>> + # name: + # matches: "" + # as: "my_external_metric" + # metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + + # resource: + # cpu: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # rate(container_cpu_usage_seconds_total{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # memory: + # containerQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(container_memory_working_set_bytes{container!="",<<.LabelMatchers>>}[3m]) + # ) + # nodeQuery: | + # sum by (<<.GroupBy>>) ( + # avg_over_time(node_memory_MemTotal_bytes{<<.LabelMatchers>>}[3m]) + # - + # avg_over_time(node_memory_MemAvailable_bytes{<<.LabelMatchers>>}[3m]) + # ) + # resources: + # overrides: + # node: + # resource: node + # namespace: + # resource: namespace + # pod: + # resource: pod + # containerLabel: container + # window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + # clusterIP: 1.2.3.4 + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Set environment variables from secrets, configmaps or by setting them as name/value +env: [] + # - name: TMP_DIR + # value: /tmp + # - name: PASSWORD + # valueFrom: + # secretKeyRef: + # name: mysecret + # key: password + # optional: false + +# Any extra arguments +extraArguments: [] + # - --tls-private-key-file=/etc/tls/tls.key + # - --tls-cert-file=/etc/tls/tls.crt + +# Additional containers to add to the pod +extraContainers: [] + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +# Annotations added to the deployment +deploymentAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS. See also dnsPolicy + enabled: false + +# When hostNetwork is enabled, you probably want to set this to ClusterFirstWithHostNet +# dnsPolicy: ClusterFirstWithHostNet + +# Deployment strategy type +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 + +certManager: + enabled: false + caCertDuration: 43800h0m0s + certDuration: 8760h0m0s diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/Chart.yaml new file mode 100644 index 000000000..2afcc3103 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,25 @@ +annotations: + artifacthub.io/license: Apache-2.0 + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts +apiVersion: v2 +appVersion: 1.8.2 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rootsandtrees@posteo.de + name: zeritti +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +type: application +version: 4.37.1 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/README.md new file mode 100644 index 000000000..ef8384410 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/README.md @@ -0,0 +1,96 @@ +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a Prometheus [Node Exporter](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +helm upgrade [RELEASE_NAME] prometheus-community/prometheus-node-exporter --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + +### From 2.x to 3.x + +Change the following: + +```yaml +hostRootFsMount: true +``` + +to: + +```yaml +hostRootFsMount: + enabled: true + mountPropagation: HostToContainer +``` + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-node-exporter +``` + +### kube-rbac-proxy + +You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint. +To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-node-exporter-read +rules: + - apiGroups: [ "" ] + resources: ["services/node-exporter-prometheus-node-exporter"] + verbs: + - get +``` + +See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100644 index 000000000..db8584def --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,29 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:9100 to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME 9100 +{{- end }} + +{{- if .Values.kubeRBACProxy.enabled}} + +kube-rbac-proxy endpoint protections is enabled: +- Metrics endpoints is now HTTPS +- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions: +``` +rules: + - apiGroups: [ "" ] + resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"] + verbs: + - get +``` +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100644 index 000000000..72a6db45a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,236 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-node-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-node-exporter.name" . }} +{{ include "prometheus-node-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-node-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if $temp_registry }} +{{- printf "%s%s:%s@%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-node-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-node-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* Sets sidecar volumeMounts */}} +{{- define "prometheus-node-exporter.sidecarVolumeMounts" -}} +{{- range $_, $mount := $.Values.sidecarVolumeMount }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- end }} +{{- range $_, $mount := $.Values.sidecarHostVolumeMounts }} +- name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} +{{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrole.yaml new file mode 100644 index 000000000..c256dba73 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrole.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: + {{- if $.Values.kubeRBACProxy.enabled }} + - apiGroups: [ "authentication.k8s.io" ] + resources: + - tokenreviews + verbs: [ "create" ] + - apiGroups: [ "authorization.k8s.io" ] + resources: + - subjectaccessreviews + verbs: [ "create" ] + {{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..653305ad9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + name: {{ template "prometheus-node-exporter.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 000000000..7f91a8d2b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,312 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + {{- $servicePort := ternary .Values.kubeRBACProxy.port .Values.service.port .Values.kubeRBACProxy.enabled }} + - name: node-exporter + image: {{ include "prometheus-node-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.hostRootFsMount.enabled }} + - --path.rootfs=/host/root + {{- if semverCompare ">=1.4.0-0" (coalesce .Values.version .Values.image.tag .Chart.AppVersion) }} + - --path.udev.data=/host/root/run/udev/data + {{- end }} + {{- end }} + - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: HOST_IP + {{- if .Values.kubeRBACProxy.enabled }} + value: 127.0.0.1 + {{- else if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if eq .Values.kubeRBACProxy.enabled false }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- end }} + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + {{- if .Values.kubeRBACProxy.enabled }} + host: 127.0.0.1 + {{- end }} + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ $servicePort }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc + {{- with .Values.hostProcFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + - name: sys + mountPath: /host/sys + {{- with .Values.hostSysFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- if .Values.hostRootFsMount.enabled }} + - name: root + mountPath: /host/root + {{- with .Values.hostRootFsMount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- with $mount.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- range .Values.sidecars }} + {{- $overwrites := dict "volumeMounts" (concat (include "prometheus-node-exporter.sidecarVolumeMounts" $ | fromYamlArray) (.volumeMounts | default list) | default list) }} + {{- $defaults := dict "image" (include "prometheus-node-exporter.image" $) "securityContext" $.Values.containerSecurityContext "imagePullPolicy" $.Values.image.pullPolicy }} + - {{- toYaml (merge $overwrites . $defaults) | nindent 10 }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy + args: + {{- if .Values.kubeRBACProxy.extraArgs }} + {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }} + {{- end }} + - --secure-listen-address=:{{ .Values.service.port}} + - --upstream=http://127.0.0.1:{{ $servicePort }}/ + - --proxy-endpoints-port={{ .Values.kubeRBACProxy.proxyEndpointsPort }} + - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml + volumeMounts: + - name: kube-rbac-proxy-config + mountPath: /etc/kube-rbac-proxy-config + imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- if .Values.kubeRBACProxy.image.sha }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}" + {{- else }} + image: "{{ $base_registry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}" + {{- end }} + ports: + - containerPort: {{ .Values.service.port}} + name: {{ .Values.kubeRBACProxy.portName }} + {{- if .Values.kubeRBACProxy.enableHostPort }} + hostPort: {{ .Values.service.port }} + {{- end }} + - containerPort: {{ .Values.kubeRBACProxy.proxyEndpointsPort }} + {{- if .Values.kubeRBACProxy.enableProxyEndpointsHostPort }} + hostPort: {{ .Values.kubeRBACProxy.proxyEndpointsPort }} + {{- end }} + name: "http-healthz" + readinessProbe: + httpGet: + scheme: HTTPS + port: {{ .Values.kubeRBACProxy.proxyEndpointsPort }} + path: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + {{- if .Values.kubeRBACProxy.resources }} + resources: + {{- toYaml .Values.kubeRBACProxy.resources | nindent 12 }} + {{- end }} + {{- if .Values.terminationMessageParams.enabled }} + {{- with .Values.terminationMessageParams }} + terminationMessagePath: {{ .terminationMessagePath }} + terminationMessagePolicy: {{ .terminationMessagePolicy }} + {{- end }} + {{- end }} + {{- with .Values.kubeRBACProxy.env }} + env: + {{- range $key, $value := $.Values.kubeRBACProxy.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.kubeRBACProxy.containerSecurityContext }} + securityContext: + {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.restartPolicy }} + restartPolicy: {{ . }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.hostRootFsMount.enabled }} + - name: root + hostPath: + path: / + {{- end }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- with $mount.type }} + type: {{ . }} + {{- end }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- if .Values.kubeRBACProxy.enabled }} + - name: kube-rbac-proxy-config + configMap: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100644 index 000000000..56b695203 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/extra-manifests.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/extra-manifests.yaml new file mode 100644 index 000000000..2b21b7106 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/networkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/networkpolicy.yaml new file mode 100644 index 000000000..825722729 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingress: + - ports: + - port: {{ .Values.service.port }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/podmonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/podmonitor.yaml new file mode 100644 index 000000000..f88da6a34 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-node-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100644 index 000000000..895731724 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.rbac.create .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ include "prometheus-node-exporter.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100644 index 000000000..333370173 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ include "prometheus-node-exporter.fullname" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ include "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100644 index 000000000..4896c84da --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.rbac.create .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.rbac.pspAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/rbac-configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/rbac-configmap.yaml new file mode 100644 index 000000000..814e11033 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/rbac-configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kubeRBACProxy.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config + namespace: {{ include "prometheus-node-exporter.namespace" . }} +data: + config-file.yaml: |+ + authorization: + resourceAttributes: + namespace: {{ template "prometheus-node-exporter.namespace" . }} + apiVersion: v1 + resource: services + subresource: {{ template "prometheus-node-exporter.fullname" . }} + name: {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 000000000..8308b7b2b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,35 @@ +{{- if .Values.service.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} +{{- end }} + type: {{ .Values.service.type }} +{{- if and (eq .Values.service.type "ClusterIP") .Values.service.clusterIP }} + clusterIP: "{{ .Values.service.clusterIP }}" +{{- end }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: {{ .Values.service.portName }} + selector: + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100644 index 000000000..462b0cda4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 000000000..abe6c73c1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.monitor-namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + metricRelabelings: + {{- with .Values.prometheus.monitor.metricRelabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 000000000..2c2705f87 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "prometheus-node-exporter.fullname" . }} + namespace: {{ include "prometheus-node-exporter.namespace" . }} + labels: + {{- include "prometheus-node-exporter.labels" . | nindent 4 }} +spec: + {{- with .Values.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: node-exporter + {{- with .Values.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ . }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml . | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: {{ include "prometheus-node-exporter.fullname" . }} + {{- with .Values.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/values.yaml new file mode 100644 index 000000000..b02d92650 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,539 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-node-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: v1.8.2 + pullPolicy: IfNotPresent + digest: "" + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +# Number of old history to retain to allow rollback +# Default Kubernetes value is set to 10 +revisionHistoryLimit: 10 + +global: + cattle: + systemDefaultRegistry: "" + + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + # + # Allow parent charts to override registry hostname + imageRegistry: "" + +# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint. +# The requests are served through the same service but requests are HTTPS. +kubeRBACProxy: + enabled: false + ## Set environment variables as name/value pairs + env: {} + # VARIABLE: value + image: + registry: docker.io + repository: rancher/mirrored-brancz-kube-rbac-proxy + tag: v0.18.0 + sha: "" + pullPolicy: IfNotPresent + + # List of additional cli arguments to configure kube-rbac-proxy + # for example: --tls-cipher-suites, --log-file, etc. + # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage + extraArgs: [] + + ## Specify security settings for a Container + ## Allows overrides and additional options compared to (Pod) securityContext + ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Specify the port used for the Node exporter container (upstream port) + port: 8100 + # Specify the name of the container port + portName: http + # Configure a hostPort. If true, hostPort will be enabled in the container and set to service.port. + enableHostPort: false + + # Configure Proxy Endpoints Port + # This is the port being probed for readiness + proxyEndpointsPort: 8888 + # Configure a hostPort. If true, hostPort will be enabled in the container and set to proxyEndpointsPort. + enableProxyEndpointsHostPort: false + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +service: + enabled: true + type: ClusterIP + clusterIP: "" + port: 9796 + targetPort: 9796 + nodePort: + portName: metrics + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + externalTrafficPolicy: "" + +# Set a NetworkPolicy with: +# ingress only on service.port +# no egress permitted +networkPolicy: + enabled: false + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + jobLabel: "" + + # List of pod labels to add to node exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: [] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Node Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +# Specify the container restart policy passed to the Node Export container +# Possible Values: Always (default)|OnFailure|Never +restartPolicy: null + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +containerSecurityContext: + readOnlyRootFilesystem: true + # capabilities: + # add: + # - SYS_TIME + +rbac: + ## If true, create & use RBAC resources + ## + create: true + ## If true, create & use Pod Security Policy resources + ## https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + pspEnabled: true + pspAnnotations: {} + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +# Mount the node's root file system (/) at /host/root in the container +hostRootFsMount: + enabled: true + # Defines how new mounts in existing mounts on the node or in the container + # are propagated to the container or node, respectively. Possible values are + # None, HostToContainer, and Bidirectional. If this field is omitted, then + # None is used. More information on: + # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation + mountPropagation: HostToContainer + +# Mount the node's proc file system (/proc) at /host/proc in the container +hostProcFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +# Mount the node's sys file system (/sys) at /host/sys in the container +hostSysFsMount: + # Possible values are None, HostToContainer, and Bidirectional + mountPropagation: "" + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to node exporter pods +podLabels: {} + +# Annotations to be added to node exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-node-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: linux + # kubernetes.io/arch: amd64 + +# Specify grace period for graceful termination of pods. Defaults to 30 if null or not specified +terminationGracePeriodSeconds: null + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +# Enable or disable container termination message settings +# https://kubernetes.io/docs/tasks/debug/debug-application/determine-reason-pod-failure/ +terminationMessageParams: + enabled: false + # If enabled, specify the path for termination messages + terminationMessagePath: /dev/termination-log + # If enabled, specify the policy for termination messages + terminationMessagePolicy: File + + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host to node-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# https://kubernetes.io/docs/concepts/storage/volumes/#hostpath-volume-types +# type: "" (Default)|DirectoryOrCreate|Directory|FileOrCreate|File|Socket|CharDevice|BlockDevice +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file; fields image,imagePullPolicy,securityContext take default value from main container +## +sidecars: [] +# - name: nvidia-dcgm-exporter +# image: nvidia/dcgm-exporter:1.4.3 +# volumeMounts: +# - name: tmp +# mountPath: /tmp + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +# - name: collector-textfiles +# mountPath: /run/prometheus +# readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +# Enable vertical pod autoscaler support for prometheus-node-exporter +verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + # updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + # updateMode: Auto + +# Extra manifests to deploy as an array +extraManifests: [] + # - | + # apiVersion: v1 + # kind: ConfigMap + # metadata: + # name: prometheus-extra + # data: + # extra-data: "value" + +# Override version of app, required if image.tag is defined and does not follow semver +version: "" diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/Chart.yaml new file mode 100644 index 000000000..22e6891fa --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rke2ControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/Chart.yaml new file mode 100644 index 000000000..901f79519 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rke2Etcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Etcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/Chart.yaml new file mode 100644 index 000000000..43b61e2bb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rke2IngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2IngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/Chart.yaml new file mode 100644 index 000000000..21778dce2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rke2Proxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Proxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/Chart.yaml new file mode 100644 index 000000000..5211d3f24 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rke2Scheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rke2Scheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/Chart.yaml new file mode 100644 index 000000000..2836d5129 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rkeControllerManager +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeControllerManager/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/Chart.yaml new file mode 100644 index 000000000..ad08b4786 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rkeEtcd +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeEtcd/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/Chart.yaml new file mode 100644 index 000000000..37691b87f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rkeIngressNginx +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeIngressNginx/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/Chart.yaml new file mode 100644 index 000000000..6e32df53d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rkeProxy +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeProxy/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/Chart.yaml new file mode 100644 index 000000000..6bca97466 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: v0.1.4-rc.4-rancher2 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +kubeVersion: '>=1.28.0-0' +name: rkeScheduler +type: application +version: 0.2.0 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/README.md new file mode 100644 index 000000000..345002f48 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/README.md @@ -0,0 +1,90 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. + +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. + +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | +| `namespaceOverride` | The namespace to install the chart | `""` + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `serviceMonitor.endpoints` | A list of endpoints that will be added to the ServiceMonitor based on the [Endpoint spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint) | `[{port: metrics}]` | +| `service.selector` | The selector that is used to populate the Service's Endpoints object. The chart will error out on rendering templating if `.Values.clients.enabled` is set alongside this field, since it is expected that this service should point to the PushProx Clients Daemonset / Deployment | `{}` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.forceHTTPSScheme` | Forces scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.enabled` | If set to true, the client will use service account credentials mounted at the configured path `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath`. This requires permissions to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath` | This is a volume mount on the pod with permissions to scrape `/metrics` endpoint of Kubernetes components | `"/var/run/secrets/kubernetes.io/serviceaccount/token"` | +| `clients.https.authenticationMethod.bearerTokenSecret.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components. This method is deprecated by the prometheus operator and may be removed in a future release | `false` | +| `clients.https.authenticationMethod.authorization.enabled` | If set to true, the client will use service account credentials to scrape `/metrics` endpoint of Kubernetes components | `false` | +| `clients.https.authenticationMethod.authorization.type` | If set, the client will use this type of authorization in its client requests for metrics | `"bearer"` | +| `clients.https.authenticationMethod.authorization.credentials.key` | If set, the client will use this key in the secret created by `clients.https.useServiceAccountCredentials` for authorization in its client requests for metrics | `"token"` | +| `clients.https.authenticationMethod.authorization.credentials.optional` | If set to false, the client will fail if the key in the secret created by `clients.https.useServiceAccountCredentials` does not exist | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.seLinuxOptions` | seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. | `""` | +| `clients.metrics.enabled` | Whether the client should publish PushProx client-specific metrics. | `false` | +| `clients.rbac.additionalRules` | Additional permissions to provide to the ServiceAccount bound to the client. This can be used to provide additional permissions for the client to scrape metrics from the k8s API. Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true | `[]` | +| `clients.deployment.enabled` | Deploys the client as a Deployment (generally used if the underlying hostNetwork Pod that is being scraped is managed by a Deployment) | `false` | +| `clients.deployment.replicas` | The number of pods the Deployment has, it should match the number of pod the hostNetwork Deployment has. Required and only used if `client.deployment.enable` is set | `0` | +| `clients.deployment.affinity` | The affinity rules that allocate the pod to the node in which the hostNetwork Deployment's pods run. Required and only used if `client.deployment.enable` is set | `{}` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | +| `kubeVersionOverrides` | A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches any of the semver constraints provided as keys on the map. On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. | `[]` + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/_helpers.tpl new file mode 100644 index 000000000..1ba509394 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,170 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "applyKubeVersionOverrides" -}} +{{- $overrides := dict -}} +{{- range $override := .Values.kubeVersionOverrides -}} +{{- if semverCompare $override.constraint $.Capabilities.KubeVersion.Version -}} +{{- $_ := mergeOverwrite $overrides $override.values -}} +{{- end -}} +{{- end -}} +{{- $_ := mergeOverwrite .Values $overrides -}} +{{- end -}} + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) (include "pushprox.namespace" .) (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.serviceAccountTokenName" -}} +{{- printf "pushprox-%s-client-service-account-token" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.endpoints" -}} +{{- $proxyURL := (include "pushProxy.proxyUrl" .) -}} +{{- $useHTTPS := .Values.clients.https.enabled -}} +{{- $setHTTPSScheme := .Values.clients.https.forceHTTPSScheme -}} +{{- $insecureSkipVerify := .Values.clients.https.insecureSkipVerify -}} +{{- $useServiceAccountCredentials := .Values.clients.https.useServiceAccountCredentials -}} +{{- $serviceAccountTokenName := (include "pushProxy.client.serviceAccountTokenName" . ) -}} +{{- $metricRelabelings := list }} +{{- $endpoints := .Values.serviceMonitor.endpoints }} +{{- if .Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- range $endpoints }} +{{- if $.Values.proxy.enabled }} +{{- $_ := set . "proxyUrl" $proxyURL }} +{{- end }} +{{- $clusterIdRelabel := dict }} +{{- $metricRelabelings := list }} +{{- if $.Values.global.cattle.clusterId }} +{{- $_ := set $clusterIdRelabel "action" "replace" }} +{{- $_ := set $clusterIdRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterIdRelabel "targetLabel" "cluster_id" }} +{{- $_ := set $clusterIdRelabel "replacement" $.Values.global.cattle.clusterId }} +{{- $metricRelabelings = append $metricRelabelings $clusterIdRelabel }} +{{- end }} +{{- $clusterNameRelabel := dict }} +{{- if $.Values.global.cattle.clusterName }} +{{- $_ := set $clusterNameRelabel "action" "replace" }} +{{- $_ := set $clusterNameRelabel "sourceLabels" (list "__address__") }} +{{- $_ := set $clusterNameRelabel "targetLabel" "cluster_name" }} +{{- $_ := set $clusterNameRelabel "replacement" $.Values.global.cattle.clusterName }} +{{- $metricRelabelings = append $metricRelabelings $clusterNameRelabel }} +{{- end }} +{{- if not (empty $metricRelabelings) }} +{{- $_ := set . "metricRelabelings" ($metricRelabelings)}} +{{- end }} +{{- if $setHTTPSScheme -}} +{{- $_ := set . "scheme" "https" }} +{{- end -}} +{{- if $useHTTPS -}} +{{- if (hasKey . "params") }} +{{- $_ := set (get . "params") "_scheme" (list "https") }} +{{- else }} +{{- $_ := set . "params" (dict "_scheme" (list "https")) }} +{{- end }} +{{- end }} +{{- if (hasKey . "tlsConfig") }} +{{- $_ := set (get . "tlsConfig") "insecureSkipVerify" $insecureSkipVerify }} +{{- else }} +{{- $_ := set . "tlsConfig" (dict "insecureSkipVerify" $insecureSkipVerify) }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenFile.enabled }} +{{- $_ := set . "bearerTokenFile" $.Values.clients.https.authenticationMethod.bearerTokenFile.bearerTokenFilePath }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.bearerTokenSecret.enabled }} +{{- $_ := set . "bearerTokenSecret" $serviceAccountTokenName }} +{{- end }} +{{- if $.Values.clients.https.authenticationMethod.authorization.enabled }} +{{- if (hasKey . "authorization") }} +{{- $_ := set (get . "authorization") "type" $.Values.clients.https.authenticationMethod.authorization.type }} +{{- $_ := set (get . "authorization") "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional) }} +{{- else }} +{{- $_ := set . "authorization" (dict "type" $.Values.clients.https.authenticationMethod.authorization.type) }} +{{- $_ := set . "authorization" (dict "credentials" (dict "name" $serviceAccountTokenName "key" $.Values.clients.https.authenticationMethod.authorization.credentials.key "optional" $.Values.clients.https.authenticationMethod.authorization.credentials.optional)) }} +{{- end }} +{{- end }} +{{- end }} +{{- toYaml $endpoints }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100644 index 000000000..a8e27c373 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,97 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- end }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- if .Values.clients.rbac.additionalRules }} +{{ toYaml .Values.clients.rbac.additionalRules }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +{{- if .Values.clients.https.useServiceAccountCredentials }} +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: {{ template "pushProxy.client.serviceAccountTokenName" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ template "pushProxy.client.name" . }} +{{- end }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 6 }} +{{- end }} + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100644 index 000000000..e8fcfb388 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,157 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +{{- if .Values.clients.deployment.enabled }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + {{- if .Values.clients.deployment.enabled }} + replicas: {{ .Values.clients.deployment.replicas }} + {{- end }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + {{- if .Values.clients.affinity }} + affinity: {{ toYaml .Values.clients.affinity | nindent 8 }} + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + {{- if .Values.clients.metrics.enabled }} + - --metrics-addr=$(PORT) + {{- end }} + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.clients.metrics.enabled }} + - name: PORT + value: :{{ .Values.clients.port }} + {{- end }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false +{{- if and .Values.global.seLinux.enabled .Values.clients.https.seLinuxOptions }} + seLinuxOptions: {{ .Values.clients.https.seLinuxOptions | toYaml | nindent 12 }} +{{- end }} + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100644 index 000000000..eefe60905 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,68 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +{{- if .Values.global.cattle.psp.enabled }} +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ include "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100644 index 000000000..723bbd6c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,57 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + {{- if .Values.global.imagePullSecretName }} + imagePullSecrets: + - name: {{ .Values.global.imagePullSecretName }} + {{- end }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100644 index 000000000..67eb2216b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,45 @@ +{{- template "applyKubeVersionOverrides" . -}} +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: {{include "pushProxy.serviceMonitor.endpoints" . | nindent 4 }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +{{- $selector := "" }} +{{- if not (kindIs "invalid" .Values.service) }} +{{- if not (kindIs "invalid" .Values.service.selector) }} +{{ if .Values.service.selector }} +{{- if .Values.clients.enabled }} +{{- required (printf "Cannot override .Values.service.selector=%s when .Values.clients.enabled=true" (toJson .Values.service.selector)) "" }} +{{- end }} +{{- $selector = (toYaml .Values.service.selector) }} +{{- end }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ default (include "pushProxy.client.labels" .) $selector | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-install-crd.yaml new file mode 100644 index 000000000..16abc2fa8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-install-crd.yaml @@ -0,0 +1,14 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install Prometheus Operator CRDs before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-psp-install.yaml new file mode 100644 index 000000000..a30c59d3b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/values.yaml new file mode 100644 index 000000000..13e981979 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/rkeScheduler/values.yaml @@ -0,0 +1,166 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + seLinux: + enabled: false + +# A list of Semver constraint strings (defined by https://github.com/Masterminds/semver) and values.yaml overrides. +# +# For each key in kubeVersionOverrides, this chart will check to see if the current Kubernetes cluster's version matches +# any of the semver constraints provided as keys on the map. +# +# On seeing a match, the default value for each values.yaml field overridden will be updated with the new value. +# +# If multiple matches are encountered (due to overlapping semver ranges), the matches will be applied in order. +# +# Notes: +# - On running a helm template, Helm generally assumes the kubeVersion is v1.20.0 +# - On running a helm install --dry-run, the correct kubeVersion should be chosen. +kubeVersionOverrides: [] +# - constraint: "< 1.21" +# values: +# metricsPort: 10252 +# clients: +# https: +# enabled: false +# insecureSkipVerify: false +# useServiceAccountCredentials: false + +namespaceOverride: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + # A list of endpoints that will be added to the ServiceMonitor based on the Endpoint spec + # Source: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#endpoint + # By default, proxyUrl and params._scheme will be overridden based on other values + endpoints: + - port: metrics + +# Configure Service that grabs scrape targets +service: + # The selector that is used to populate the Service's Endpoints object. + # The chart will error out on rendering templating if .Values.clients.enabled is set alongside this field, + # since it is expected that this service should point to the PushProx Clients Daemonset / Deployment + selector: {} + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # Does the client require requests be sent to http or https? + forceHTTPSScheme: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # Configuration for authentication to metrics via https endpoint + authenticationMethod: + # Reads token from defined file in container + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenFile: + enabled: false + bearerTokenFilePath: "/var/run/secrets/kubernetes.io/serviceaccount/token" + # Reads token from defined secret in namespace + # This function is deprecated in the prometheus operator api and may be removed in a future version + bearerTokenSecret: + enabled: false + # Reads token from defined secret in namespace + authorization: + enabled: false + type: "bearer" + credentials: + key: "token" + optional: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + # seLinuxOptions to be passed into the container that copies certs. Should define a container with permissions to read the files in the certDir provided on the host. + # Required and only used if `clients.https.enabled` is set and `clients.https.certDir` is provided. + seLinuxOptions: {} + + metrics: + # Whether the client should publish PushProx client-specific metrics to .Values.clients.port + enabled: false + + rbac: + # Additional permissions to provide to the ServiceAccount bound to the client + # This can be used to provide additional permissions for the client to scrape metrics from the k8s API + # Only enabled if clients.https.enabled and clients.https.useServiceAccountCredentials are true + additionalRules: [] + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + affinity: {} + + image: + repository: rancher/pushprox-client + tag: v0.1.4-rc.4-rancher2-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + + # The default intention of rancher-pushprox clients is to scrape hostNetwork metrics across all nodes. + # This can be used to scrape internal Kubernetes components or DaemonSets of hostNetwork Pods in + # situations where a cloud provider firewall prevents Pod-To-Host communication but not Pod-To-Pod. + # However, if the underlying hostNetwork Pod that is being scraped is managed by a Deployment, + # this advanced option enables users to deploy the client as a Deployment instead of a DaemonSet. + # If a user deploys this feature and the underlying Deployment's number of replicas changes, the user will + # be responsible for upgrading this chart accordingly to the right number of replicas. + deployment: + enabled: false + replicas: 0 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.4-rc.4-rancher2-proxy + command: ["pushprox-proxy"] diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/Chart.yaml new file mode 100644 index 000000000..b2fa9e7be --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/Chart.yaml @@ -0,0 +1,24 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/kube-version: '>= 1.28.0-0 < 1.32.0-0' + catalog.cattle.io/os: windows + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-windows-exporter +apiVersion: v2 +appVersion: 0.25.1 +description: A Helm chart for prometheus windows-exporter +home: https://github.com/prometheus-community/windows_exporter/ +keywords: +- windows-exporter +- windows +- prometheus +- exporter +maintainers: +- email: github@jkroepke.de + name: jkroepke +name: windowsExporter +sources: +- https://github.com/prometheus-community/windows_exporter/ +type: application +version: 0.3.1 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/README.md b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/README.md new file mode 100644 index 000000000..1da1c64e1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/README.md @@ -0,0 +1,42 @@ +# Prometheus `Windows Exporter` + +Prometheus exporter for hardware and OS metrics exposed by Windows kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [`Windows Exporter`](http://github.com/prometheus-community/windows_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repository Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +helm install [RELEASE_NAME] prometheus-community/prometheus-windows-exporter +``` + +_See [configuration](#configuring) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +helm show values prometheus-community/prometheus-windows-exporter +``` diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/scripts/configure-firewall.ps1 b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/scripts/configure-firewall.ps1 new file mode 100644 index 000000000..9cbed7112 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/scripts/configure-firewall.ps1 @@ -0,0 +1,31 @@ +$ErrorActionPreference = 'Continue' + +function CheckFirewallRuleError { + # We hit an error. This can happen for a number of reasons, including if the rule already exists + if ($error[0]) { + if (($error[0].Exception.NativeErrorCode) -and ($error[0].Exception.NativeErrorCode.ToString() -eq "AlreadyExists")) { + # Previous versions of monitoring may have already created this Firewall Rule + # Because of this, if the rule alreadys exists there is no need to delete and recreate it. + Write-Host "Detected Existing Firewall Rule, Nothing To Do" + } else { + Write-Host "Error Encountered Setting Up Required Firewall Rule" + $error[0].Exception + exit 1 + } + } +} + +Write-Host "Attempting To Configure Firewall Rules For Ports 9796, 10250" + +# This is the exact same firewall rule that has historically been created by rancher-wins +# https://github.com/rancher/wins/blob/91f670c47f19c6d9fe97d8f66a695d3081ad994f/pkg/apis/process_service_mgmt.go#L149 +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-9796 -Name rancher-wins-windows-exporter-TCP-9796 -Action Allow -Protocol TCP -LocalPort 9796 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Node Exporter Firewall Rule Successfully Created" + +# This rule is required in order to have the Rancher UI display node metrics in the 'Nodes' tab of the cluster explorer +New-NetFirewallRule -DisplayName rancher-wins-windows-exporter-TCP-10250 -Name rancher-wins-windows-exporter-TCP-10250 -Action Allow -Protocol TCP -LocalPort 10250 -Enabled True -PolicyStore ActiveStore +CheckFirewallRuleError +Write-Host "Windows Prometheus Metrics Firewall Rule Successfully Created" + +Write-Host "All Firewall Rules Successfully Configured" diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/_helpers.tpl new file mode 100644 index 000000000..c9a5d6db8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "prometheus-windows-exporter.fullname" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.renamedMetricsRelabeling" -}} +{{- range $original, $new := (include "windowsExporter.renamedMetrics" . | fromJson) -}} +- sourceLabels: [__name__] + regex: {{ $original }} + replacement: '{{ $new }}' + targetLabel: __name__ +{{ end -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "prometheus-windows-exporter.fullname" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +{{- define "windowsExporter.renamedMetrics" -}} +{{- $renamed := dict -}} +{{/* v0.15.0 */}} +{{- $_ := set $renamed "windows_mssql_transactions_active_total" "windows_mssql_transactions_active" -}} +{{/* v0.16.0 */}} +{{- $_ := set $renamed "windows_adfs_ad_login_connection_failures" "windows_adfs_ad_login_connection_failures_total" -}} +{{- $_ := set $renamed "windows_adfs_certificate_authentications" "windows_adfs_certificate_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_device_authentications" "windows_adfs_device_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_extranet_account_lockouts" "windows_adfs_extranet_account_lockouts_total" -}} +{{- $_ := set $renamed "windows_adfs_federated_authentications" "windows_adfs_federated_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_passport_authentications" "windows_adfs_passport_authentications_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_failed" "windows_adfs_password_change_failed_total" -}} +{{- $_ := set $renamed "windows_adfs_password_change_succeeded" "windows_adfs_password_change_succeeded_total" -}} +{{- $_ := set $renamed "windows_adfs_token_requests" "windows_adfs_token_requests_total" -}} +{{- $_ := set $renamed "windows_adfs_windows_integrated_authentications" "windows_adfs_windows_integrated_authentications_total" -}} +{{- $_ := set $renamed "windows_net_packets_outbound_errors" "windows_net_packets_outbound_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_discarded" "windows_net_packets_received_discarded_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_errors" "windows_net_packets_received_errors_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_total" "windows_net_packets_received_total_total" -}} +{{- $_ := set $renamed "windows_net_packets_received_unknown" "windows_net_packets_received_unknown_total" -}} +{{- $_ := set $renamed "windows_dns_memory_used_bytes_total" "windows_dns_memory_used_bytes" -}} +{{- $renamed | toJson -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-windows-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-windows-exporter.labels" -}} +helm.sh/chart: {{ include "prometheus-windows-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ include "prometheus-windows-exporter.name" . }} +{{ include "prometheus-windows-exporter.selectorLabels" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +{{- with .Values.podLabels }} +{{ toYaml . }} +{{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-windows-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-windows-exporter.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-windows-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-windows-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "prometheus-windows-exporter.image" -}} +{{- if .Values.image.sha }} +{{- fail "image.sha forbidden. Use image.digest instead" }} +{{- else if .Values.image.digest }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s@%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- else }} +{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) .Values.image.digest }} +{{- end }} +{{- else }} +{{- if .Values.global.cattle.systemDefaultRegistry }} +{{- printf "%s/%s:%s" .Values.global.cattle.systemDefaultRegistry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- else }} +{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-windows-exporter.namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-windows-exporter.monitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.monitor.namespace }} +{{- .Values.prometheus.monitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} + +{{/* +Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets}) +*/}} +{{- define "prometheus-windows-exporter.imagePullSecrets" -}} +{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Create the namespace name of the pod monitor +*/}} +{{- define "prometheus-windows-exporter.podmonitor-namespace" -}} +{{- if .Values.namespaceOverride }} +{{- .Values.namespaceOverride }} +{{- else }} +{{- if .Values.prometheus.podMonitor.namespace }} +{{- .Values.prometheus.podMonitor.namespace }} +{{- else }} +{{- .Release.Namespace }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Sets default scrape limits for podmonitor */}} +{{- define "podmonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/config.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/config.yaml new file mode 100644 index 000000000..25f1fa69c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + config.yml: | + {{- .Values.config | nindent 4 }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/daemonset.yaml new file mode 100644 index 000000000..be7feb3ed --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,200 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- with .Values.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "windowsExporter.labels" . | nindent 8 }} + spec: + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + initContainers: + - name: configure-firewall + image: {{ include "prometheus-windows-exporter.image" . }} + command: + - C:\WINDOWS\System32\WindowsPowerShell\v1.0\powershell.exe + args: ["-f", "scripts/configure-firewall.ps1"] + volumeMounts: + - mountPath: /scripts + name: exporter-scripts + {{- with .Values.extraInitContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "prometheus-windows-exporter.fullname" . }} + containers: + - name: windows-exporter + image: {{ include "prometheus-windows-exporter.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --config.file=%CONTAINER_SANDBOX_MOUNT_POINT%/config.yml + - --collector.textfile.directories=%CONTAINER_SANDBOX_MOUNT_POINT% + - --web.listen-address=:{{ .Values.service.port }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- range $key, $value := .Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + hostPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.livenessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.livenessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + readinessProbe: + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + httpGet: + httpHeaders: + {{- range $_, $header := .Values.readinessProbe.httpGet.httpHeaders }} + - name: {{ $header.name }} + value: {{ $header.value }} + {{- end }} + path: / + port: {{ .Values.service.port }} + scheme: {{ upper .Values.readinessProbe.httpGet.scheme }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config.yml + subPath: config.yml + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: {{ .Values.hostPID }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: exporter-scripts + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + - name: config + configMap: + name: {{ include "prometheus-windows-exporter.fullname" . }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/podmonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/podmonitor.yaml new file mode 100644 index 000000000..bbb6c3934 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/podmonitor.yaml @@ -0,0 +1,91 @@ +{{- if .Values.prometheus.podMonitor.enabled }} +apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: PodMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.podmonitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }} + {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }} + selector: + matchLabels: + {{- with .Values.prometheus.podMonitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "prometheus-windows-exporter.selectorLabels" . | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "prometheus-windows-exporter.namespace" . }} + {{- with .Values.prometheus.podMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: {{ .Values.service.portName }} + {{- with .Values.prometheus.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.bearerTokenSecret }} + bearerTokenSecret: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.authorization }} + authorization: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.oauth2 }} + oauth2: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorTimestamps }} + honorTimestamps: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }} + filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }} + followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }} + {{- with .Values.prometheus.podMonitor.params }} + params: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/scriptConfig.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/scriptConfig.yaml new file mode 100644 index 000000000..f514c8161 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/scriptConfig.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }}-scripts + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} + diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/service.yaml new file mode 100644 index 000000000..267b796f6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" $ | nindent 4 }} + {{- if or .Values.prometheus.monitor.enabled .Values.prometheus.podMonitor.enabled }} + {{- with .Values.service.annotations }} + annotations: + {{- unset . "prometheus.io/scrape" | toYaml | nindent 4 }} + {{- end }} + {{- else }} + annotations: + prometheus.io/scrape: "true" + {{- with .Values.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.port }} + protocol: TCP + appProtocol: http + name: {{ .Values.service.portName }} + selector: + {{- include "windowsExporter.labels" . | nindent 4 }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/serviceaccount.yaml new file mode 100644 index 000000000..14c1c4680 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-windows-exporter.serviceAccountName" . }} + namespace: {{ include "prometheus-windows-exporter.namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }} +imagePullSecrets: + {{- include "prometheus-windows-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100644 index 000000000..2effc0775 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: {{ .Values.prometheus.monitor.apiVersion | default "monitoring.coreos.com/v1" }} +kind: ServiceMonitor +metadata: + name: {{ include "prometheus-windows-exporter.fullname" . }} + namespace: {{ include "prometheus-windows-exporter.monitor-namespace" . }} + labels: + {{- include "windowsExporter.labels" . | nindent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }} + {{- with .Values.prometheus.monitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- with .Values.prometheus.monitor.selectorOverride }} + {{- toYaml . | nindent 6 }} + {{- else }} + {{- include "windowsExporter.labels" . | nindent 6 }} + {{- end }} + {{- with .Values.prometheus.monitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.basicAuth }} + basicAuth: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + metricRelabelings: +{{- include "windowsExporter.renamedMetricsRelabeling" . | nindent 6 -}} + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode + relabelings: + - separator: ':' + sourceLabels: + - __meta_kubernetes_pod_host_ip + - __meta_kubernetes_pod_container_port_number + targetLabel: instance +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/values.yaml new file mode 100644 index 000000000..6f76b5ea9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/charts/windowsExporter/values.yaml @@ -0,0 +1,367 @@ +# Default values for prometheus-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + registry: docker.io + repository: rancher/mirrored-prometheus-windows-exporter + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + os: "windows" + tag: "0.25.1" + pullPolicy: IfNotPresent + digest: "" + +config: |- + collectors: + enabled: '[defaults],tcp,memory,container' + +imagePullSecrets: [] +# - name: "image-pull-secret" +nameOverride: "" +fullnameOverride: "" + +global: + # To help compatibility with other charts which use global.imagePullSecrets. + # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). + # global: + # imagePullSecrets: + # - name: pullSecret1 + # - name: pullSecret2 + # or + # global: + # imagePullSecrets: + # - pullSecret1 + # - pullSecret2 + imagePullSecrets: [] + cattle: + systemDefaultRegistry: "" + +service: + type: ClusterIP + port: 9796 + nodePort: + portName: windows-metrics + annotations: {} + +# Additional environment variables that will be passed to the daemonset +env: {} +## env: +## VARIABLE: value + +prometheus: + monitor: + enabled: true + additionalLabels: {} + namespace: "" + + jobLabel: "component" + + # List of pod labels to add to windows exporter metrics + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor + podTargetLabels: ["component"] + + scheme: http + basicAuth: {} + bearerTokenFile: + tlsConfig: {} + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Override serviceMonitor selector + ## + selectorOverride: {} + + ## Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + ## + attachMetadata: + node: false + + relabelings: [] + metricRelabelings: [] + interval: "" + scrapeTimeout: 10s + ## prometheus.monitor.apiVersion ApiVersion for the serviceMonitor Resource(defaults to "monitoring.coreos.com/v1") + apiVersion: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + # PodMonitor defines monitoring for a set of pods. + # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + # Using a PodMonitor may be preferred in some environments where there is very large number + # of Windows Exporter endpoints (1000+) behind a single service. + # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor, + # the time series resulting from the configuration through PodMonitor may have different labels. + # For instance, there will not be the service label any longer which might + # affect PromQL queries selecting that label. + podMonitor: + enabled: false + # Namespace in which to deploy the pod monitor. Defaults to the release namespace. + namespace: "" + # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus + additionalLabels: {} + # release: kube-prometheus-stack + # PodTargetLabels transfers labels of the Kubernetes Pod onto the target. + podTargetLabels: [] + # apiVersion defaults to monitoring.coreos.com/v1. + apiVersion: "" + # Override pod selector to select pod objects. + selectorOverride: {} + # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above. + attachMetadata: + node: false + # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name. + jobLabel: "" + + # Scheme/protocol to use for scraping. + scheme: "http" + # Path to scrape metrics at. + path: "/metrics" + + # BasicAuth allow an endpoint to authenticate over basic authentication. + # More info: https://prometheus.io/docs/operating/configuration/#endpoint + basicAuth: {} + # Secret to mount to read bearer token for scraping targets. + # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator. + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core + bearerTokenSecret: {} + # TLS configuration to use when scraping the endpoint. + tlsConfig: {} + # Authorization section for this endpoint. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization + authorization: {} + # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer. + # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2 + oauth2: {} + + # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint. + proxyUrl: "" + # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used. + interval: "" + # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used. + scrapeTimeout: "" + # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. + honorTimestamps: true + # HonorLabels chooses the metric’s labels on collisions with target labels. + honorLabels: true + # Whether to enable HTTP2. Default false. + enableHttp2: "" + # Drop pods that are not running. (Failed, Succeeded). + # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + filterRunning: "" + # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false. + followRedirects: "" + # Optional HTTP URL parameters + params: {} + + # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds + # relabelings for a few standard Kubernetes fields. The original scrape job’s name + # is available via the __tmp_prometheus_job_name label. + # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # MetricRelabelConfigs to apply to samples before ingestion. + metricRelabelings: [] + + # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + sampleLimit: 0 + # TargetLimit defines a limit on the number of scraped targets that will be accepted. + targetLimit: 0 + # Per-scrape limit on number of labels that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelLimit: 0 + # Per-scrape limit on length of labels name that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelNameLengthLimit: 0 + # Per-scrape limit on length of labels value that will be accepted for a sample. + # Only valid in Prometheus versions 2.27.0 and newer. + labelValueLengthLimit: 0 + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m +# memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + annotations: {} + imagePullSecrets: [] + automountServiceAccountToken: false + +securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + +rbac: + ## If true, create & use RBAC resources + ## + create: true + +# Expose the service to the host network +hostNetwork: true + +# Share the host process ID namespace +hostPID: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to windows exporter pods +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + +# Extra labels to be added to windows exporter pods +podLabels: {} + +# Annotations to be added to windows exporter daemonset +daemonsetAnnotations: {} + +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + +# Custom DNS configuration to be added to prometheus-windows-exporter pods +dnsConfig: {} +# nameservers: +# - 1.2.3.4 +# searches: +# - ns1.svc.cluster-domain.example +# - my.dns.search.suffix +# options: +# - name: ndots +# value: "2" +# - name: edns0 + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: + kubernetes.io/os: windows + # kubernetes.io/arch: amd64 + +tolerations: + - effect: NoSchedule + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.service.services-where +# - "Name LIKE 'sql%'" + +## Additional mounts from the host to windows-exporter container +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: +secrets: [] +# - name: +# mountPath: +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false + +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional InitContainers to initialize the pod +## +extraInitContainers: [] + +## Liveness probe +## +livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + +## Readiness probe +## +readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: [] + scheme: http + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/nginx.json new file mode 100644 index 000000000..565352235 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/nginx.json @@ -0,0 +1,1445 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "$datasource", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "$datasource", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/request-handling-performance.json new file mode 100644 index 000000000..156e33123 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,963 @@ +{ + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "$datasource", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster-nodes.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster-nodes.json new file mode 100644 index 000000000..d1cc3b70a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster-nodes.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m] ({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m])) by (instance)", + "interval": "", + "legendFormat": "Load[1m] ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m])) by (instance)", + "interval": "", + "legendFormat": "Load[5m] ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m])) by (instance)", + "interval": "", + "legendFormat": "Load[15m] ({{instance}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) by (instance) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes) by (instance) ", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Read ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Write ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Errors ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Errors ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Dropped ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Dropped ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) by (instance) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Transmit Total ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Receive Total ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster (Nodes)", + "uid": "rancher-cluster-nodes-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster.json new file mode 100644 index 000000000..ec977f55d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/cluster/rancher-cluster.json @@ -0,0 +1,776 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load1 OR avg_over_time(windows_system_processor_queue_length[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "A" + }, + { + "expr": "sum(node_load5 OR avg_over_time(windows_system_processor_queue_length[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "B" + }, + { + "expr": "sum(node_load15 OR avg_over_time(windows_system_processor_queue_length[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes OR windows_os_physical_memory_free_bytes) / sum(node_memory_MemTotal_bytes OR windows_cs_physical_memory_bytes)", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}))", + "legendFormat": "Total", + "interval": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*'}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Cluster", + "uid": "rancher-cluster-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundle.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundle.json new file mode 100644 index 000000000..698f48aee --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundle.json @@ -0,0 +1,246 @@ +{ + "description": "Bundle", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_out_of_sync{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Out of Sync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_err_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_pending{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundle_wait_applied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Bundles", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Bundle", + "uid": "fleet-bundle" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundledeployment.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundledeployment.json new file mode 100644 index 000000000..c81f7a621 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/bundledeployment.json @@ -0,0 +1,219 @@ +{ + "description": "BundleDeployment", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"}) / sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\"})" + } + ], + "title": "Ready BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"WaitApplied\"})", + "legendFormat": "Wait Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"ErrApplied\"})", + "legendFormat": "Err Applied" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"OutOfSync\"})", + "legendFormat": "OutOfSync" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Pending\"})", + "legendFormat": "Pending" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_bundledeployment_state{cluster_namespace=~\"$namespace\",state=\"Modified\"})", + "legendFormat": "Modified" + } + ], + "title": "BundleDeployments", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_bundledeployment_state, cluster_namespace)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / BundleDeployment", + "uid": "fleet-bundledeployment" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/cluster.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/cluster.json new file mode 100644 index 000000000..73bdea483 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/cluster.json @@ -0,0 +1,484 @@ +{ + "description": "Cluster", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_desired_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_ready_git_repos{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Git Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_desiredready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_resources_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"}) / sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"Ready\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"NotReady\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_state{exported_namespace=\"$namespace\",name=~\"$name\",state=\"WaitCheckIn\"})", + "legendFormat": "Wait Check In" + } + ], + "title": "Clusters", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_desired_ready_git_repos, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_desired_ready_git_repos{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Cluster", + "uid": "fleet-cluster" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/clustergroup.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/clustergroup.json new file mode 100644 index 000000000..ce3df87b2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/clustergroup.json @@ -0,0 +1,468 @@ +{ + "description": "ClusterGroup", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_bundle_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Bundles", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"}) - sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})) / sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_non_ready_cluster_count{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Non Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 26 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 26 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_notready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_orphaned{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Orphaned" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_cluster_group_resource_count_waitapplied{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Wait Applied" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_cluster_group_bundle_desired_ready, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_cluster_group_bundle_desired_ready{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / ClusterGroup", + "uid": "fleet-cluster-group" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/controller-runtime.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/controller-runtime.json new file mode 100644 index 000000000..23a81f2a8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/controller-runtime.json @@ -0,0 +1,454 @@ +{ + "description": "Controller Runtime", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{controller}} {{instance}}" + } + ], + "title": "Number of Workers in Use", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Reconciliation Error Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "Total Reconciliation Count per Controller", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}", + "legendFormat": "{{instance}} {{pod}}" + } + ], + "title": "WorkQueue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P50", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P90", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 7, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds for Items Stay in Queue (before being requested) P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 8, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Add Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 9, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Unfinished Seconds", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 10, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P50 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 50th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 11, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P90 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 90th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 12, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))", + "legendFormat": "P99 {{name}}" + } + ], + "title": "Seconds Processing Items from WorkQueue - 99th Percentile", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": null, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 13, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)", + "legendFormat": "{{name}} {{instance}}" + } + ], + "title": "Work Queue Retries Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(controller_runtime_reconcile_total, namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "job", + "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / Controller-Runtime", + "uid": "fleet-controller-runtime" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/gitrepo.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/gitrepo.json new file mode 100644 index 000000000..1a50c2937 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/fleet/gitrepo.json @@ -0,0 +1,325 @@ +{ + "description": "GitRepo", + "graphTooltip": 1, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 0 + }, + "id": 1, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 0 + }, + "id": 2, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_desired_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_ready_clusters{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + } + ], + "title": "Clusters", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 13 + }, + "id": 4, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"}) / sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})" + } + ], + "title": "Ready Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 5, + "w": 17, + "x": 7, + "y": 13 + }, + "id": 5, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": null + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_desired_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Desired Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_not_ready{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Not Ready" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_missing{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Missing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_modified{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Modified" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(fleet_gitrepo_resources_unknown{exported_namespace=\"$namespace\",name=~\"$name\"})", + "legendFormat": "Unknown" + } + ], + "title": "Resources", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "name": "namespace", + "query": "label_values(fleet_gitrepo_desired_ready_clusters, exported_namespace)", + "refresh": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "includeAll": true, + "name": "name", + "query": "label_values(fleet_gitrepo_desired_ready_clusters{exported_namespace=~\"$namespace\"}, name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Fleet / GitRepo", + "uid": "fleet-gitrepo" +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/home/rancher-default-home.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/home/rancher-default-home.json new file mode 100644 index 000000000..3fce20756 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/home/rancher-default-home.json @@ -0,0 +1,1290 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[5m])))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode!=\"idle\"}[5m]))", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_cpu_cores{}) OR sum(kube_node_status_allocatable{resource=\"cpu\",unit=\"core\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "CPU Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kube_node_status_allocatable_memory_bytes{}) OR sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Memory Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) - sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Disk Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\",mode=\"idle\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", mode=\"idle\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum({__name__=~\"node_memory_MemAvailable_bytes|windows_os_physical_memory_free_bytes\"}) by (instance) / sum({__name__=~\"node_memory_MemTotal_bytes|windows_cs_physical_memory_bytes\"}) by (instance))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(1 - ((sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"} OR on() vector(0)))) / ((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) OR on() vector(0)) + (sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) OR on() vector(0)))) * 100", + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "(1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "B" + }, + { + "expr": "(1 - (sum(windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) / sum(windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\"}) by (instance)) * 100", + "hide": false, + "legendFormat": "{{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "rancher-home-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd-nodes.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd-nodes.json new file mode 100644 index 000000000..8af4b81ce --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd-nodes.json @@ -0,0 +1,687 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic In ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Client Traffic Out ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes) by (instance)", + "interval": "", + "legendFormat": "DB Size ({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Watch Streams ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) by (instance)", + "interval": "", + "legendFormat": "Lease Watch Stream ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Committed ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Applied ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "Proposal Failed ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending) by (instance)", + "interval": "", + "legendFormat": "Proposal Pending ({{instance}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Rate ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval])) by (instance)", + "interval": "", + "legendFormat": "RPC Failure Rate ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync ({{instance}})", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync ({{instance}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd (Nodes)", + "uid": "rancher-etcd-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd.json new file mode 100644 index 000000000..0c058cafb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-etcd.json @@ -0,0 +1,669 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 33, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_client_grpc_received_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic In", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_network_client_grpc_sent_bytes_total{job=\"kube-etcd\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Client Traffic Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GRPC Client Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(etcd_mvcc_db_total_size_in_bytes)", + "interval": "", + "legendFormat": "DB Size", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Watch Streams", + "refId": "A" + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "interval": "", + "legendFormat": "Lease Watch Stream", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_committed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Committed", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Applied", + "refId": "B" + }, + { + "expr": "sum(rate(etcd_server_proposals_failed_total[$__rate_interval]))", + "interval": "", + "legendFormat": "Proposal Failed", + "refId": "C" + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "interval": "", + "legendFormat": "Proposal Pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Rate", + "refId": "A" + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "RPC Failure Rate", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "WAL fsync", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[$__rate_interval])) by (instance, le))", + "interval": "", + "legendFormat": "DB fsync", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / etcd", + "uid": "rancher-etcd-1", + "version": 4 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components-nodes.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components-nodes.json new file mode 100644 index 000000000..b31358eaa --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components-nodes.json @@ -0,0 +1,527 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "links": [], + "panels": [ + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (instance, code)", + "interval": "", + "legendFormat": "{{code}}({{instance}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (instance, name)", + "interval": "", + "legendFormat": "Deployment Depth ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (instance, name)", + "interval": "", + "legendFormat": "Volumes Depth ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicaSet Depth ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (instance, name)", + "interval": "", + "legendFormat": "Service Depth ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (instance, name)", + "interval": "", + "legendFormat": "ServiceAccount Depth ({{instance}})", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (instance, name)", + "interval": "", + "legendFormat": "Endpoint Depth ({{instance}})", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (instance, name)", + "interval": "", + "legendFormat": "DaemonSet Depth ({{instance}})", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (instance, name)", + "interval": "", + "legendFormat": "StatefulSet Depth ({{instance}})", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (instance, name)", + "interval": "", + "legendFormat": "ReplicationManager Depth ({{instance}})", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{instance}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"}) by (instance)", + "interval": "", + "legendFormat": "Reading ({{instance}})", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"}) by (instance)", + "interval": "", + "legendFormat": "Waiting ({{instance}})", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"}) by (instance)", + "interval": "", + "legendFormat": "Writing ({{instance}})", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Accepted ({{instance}})", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval]))) by (instance)", + "interval": "", + "legendFormat": "Handled ({{instance}})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components (Nodes)", + "uid": "rancher-k8s-components-nodes-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components.json new file mode 100644 index 000000000..44cf97f9f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/k8s/rancher-k8s-components.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 31, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[$__rate_interval])) by (code)", + "interval": "", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Server Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]({{instance}})" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"deployment\"}) by (name)", + "interval": "", + "legendFormat": "Deployment Depth", + "refId": "A" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"volumes\"}) by (name)", + "interval": "", + "legendFormat": "Volumes Depth", + "refId": "B" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicaset\"}) by (name)", + "interval": "", + "legendFormat": "Replicaset Depth", + "refId": "C" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"service\"}) by (name)", + "interval": "", + "legendFormat": "Service Depth", + "refId": "D" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"serviceaccount\"}) by (name)", + "interval": "", + "legendFormat": "ServiceAccount Depth", + "refId": "E" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"endpoint\"}) by (name)", + "interval": "", + "legendFormat": "Endpoint Depth", + "refId": "F" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"daemonset\"}) by (name)", + "interval": "", + "legendFormat": "DaemonSet Depth", + "refId": "G" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"statefulset\"}) by (name)", + "interval": "", + "legendFormat": "StatefulSet Depth", + "refId": "H" + }, + { + "expr": "sum(workqueue_depth{component=\"kube-controller-manager\", name=\"replicationmanager\"}) by (name)", + "interval": "", + "legendFormat": "ReplicationManager Depth", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Controller Manager Queue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_status_scheduled{condition=\"false\"})", + "interval": "", + "legendFormat": "Failed To Schedule", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pod Scheduling Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"reading\"})", + "interval": "", + "legendFormat": "Reading", + "refId": "A" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"waiting\"})", + "interval": "", + "legendFormat": "Waiting", + "refId": "B" + }, + { + "expr": "sum(nginx_ingress_controller_nginx_process_connections{state=\"writing\"})", + "interval": "", + "legendFormat": "Writing", + "refId": "C" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"accepted\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Accepted", + "refId": "D" + }, + { + "expr": "sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state=\"handled\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Handled", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ingress Controller Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Kubernetes Components", + "uid": "rancher-k8s-components-1", + "version": 5 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentbit.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentbit.json new file mode 100644 index 000000000..b00582c8a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentbit.json @@ -0,0 +1,760 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:7", + "builtIn": 1, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitoring of the logging-stack", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13042, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "General", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1802", + "alias": "/Error.*/", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "\nsum(rate(fluentbit_output_retries_total[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Retry rate", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_output_errors_total[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Error rate", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentbit output error/retry rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1697", + "format": "ops", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1698", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Input and output total rates", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_input_records_total[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "input", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_output_proc_records_total[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "output", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentbit input/output rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_output_retries_total[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current retry count", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_output_errors_total[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current error count", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_input_records_total[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input records total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(fluentbit_filter_drop_records_total[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Dropped records total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "logging", + "fluentbit" + ], + "templating": { + "list": [ + { + "hide": 2, + "name": "DS_PROMETHEUS", + "query": "prometheus", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "2024-08-20T15:06:03.311Z", + "to": "2024-08-20T21:06:03.311Z" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Rancher / Fluentbit", + "uid": "rancher-logging-fluentbit", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentd.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentd.json new file mode 100644 index 000000000..8861425ce --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/logging/fluentd.json @@ -0,0 +1,3221 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:7", + "builtIn": 1, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitoring of the logging-stack", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13042, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "General", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "If you see errors then you probbaly have serious issues with log processing, see https://docs.fluentd.org/buffer#handling-unrecoverable-errors\n\nRetries are normal but should occur only from time to time, otherwise check for network errors or destination is too slow and requires additional tuning per given provider.", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1802", + "alias": "/Error.*/", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_retry_count[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Retry rate", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_num_errors[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Error rate", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output error/retry rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1697", + "format": "ops", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1698", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Input and output total rates", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "input", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_write_count[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "output", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd input/output rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "This should not reach 0 otherwise logs are blocked from processing or even dropped", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(fluentd_output_status_buffer_available_space_ratio)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "lowest across all hosts", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output status buffer available space ratio", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:918", + "decimals": 0, + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:919", + "decimals": 0, + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "total flush time", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:906", + "alias": "count", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_flush_time_count[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_slow_flush_count[1m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output status flush time count rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:400", + "decimals": 0, + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:401", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current total size of stage and queue buffers.\nfluentd_output_status_buffer_total_bytes", + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_total_bytes) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of stage and queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:321", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:322", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_queue_length)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Fluentd output buffer queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1460", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1461", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 17, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_input_status_num_records_total", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m])) ", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_input_status_num_records_total", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m])) by (hostname)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per hostname", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_input_status_num_records_total", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m])) by (namespace)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per namespace", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_input_status_num_records_total", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per instance", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Input details", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 59, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_input_status_num_records_total", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_input_status_num_records_total[1m])) by (tag)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tag}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Input entries rate per tag", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Input details (warning, very slow!)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 41, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_stage_length", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_stage_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current length of stage buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_stage_byte_size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_stage_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of stage buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer stage", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 43, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max_over_time(fluentd_output_status_buffer_queue_length[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Maximum buffer length in last 1min", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 39 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max_over_time(fluentd_output_status_buffer_total_bytes[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Maximum buffer bytes in last 1min", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_queue_length", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_queue_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current length of queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_queue_byte_size", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 45 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_queue_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total size of queue buffers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer queue", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 46, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_available_space_ratio", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_available_space_ratio) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Ratio of available space in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer space", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 53, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_retry_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 59 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_retry_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current retry counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_emit_records", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 59 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_emit_records[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current emit records", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_emit_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 65 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_emit_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current emit counts rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_rollback_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 65 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_rollback_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current rollback counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_write_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_write_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current write counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_slow_flush_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 71 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_slow_flush_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current slow flush counts", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_retry_wait", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 77 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_retry_wait[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current retry wait", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_flush_time_count", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 77 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_flush_time_count[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total flush time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer retries", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 57, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_num_errors", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 84 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fluentd_output_status_num_errors[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current number of errors rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer errors", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 55, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 91 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "fluentd_output_status_buffer_newest_timekey - fluentd_output_status_buffer_oldest_timekey", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Timekey diff", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_newest_timekey", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 91 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_newest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Newest timekey in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "fluentd_output_status_buffer_oldest_timekey", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 97 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fluentd_output_status_buffer_oldest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oldest timekey in buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:250", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:251", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Buffer timekeys", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "fluentd", + "logging" + ], + "templating": { + "list": [ + { + "hide": 2, + "name": "DS_PROMETHEUS", + "query": "prometheus", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Rancher / Fluentd", + "uid": "rancher-logging-fluentd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node-detail.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node-detail.json new file mode 100644 index 000000000..920fb94cf --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node-detail.json @@ -0,0 +1,805 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": { + "{{mode}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\"}[$__rate_interval])) by (mode)", + "interval": "", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / (node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) by (device))", + "interval": "", + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Read ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Write ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Errors ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + }, + { + "expr": "sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Errors ({{device}})", + "refId": "C" + }, + { + "expr": "sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Dropped ({{device}})", + "refId": "D" + }, + { + "expr": "sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Dropped ({{device}})", + "refId": "E" + }, + { + "expr": "sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) by (device) OR sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{device}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Transmit Total ({{device}})", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) by (device)", + "interval": "", + "legendFormat": "Receive Total ({{device}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node (Detail)", + "uid": "rancher-node-detail-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node.json new file mode 100644 index 000000000..367df3cc9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/nodes/rancher-node.json @@ -0,0 +1,792 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(irate({__name__=~\"node_cpu_seconds_total|windows_cpu_time_total\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load[5m]" + }, + "properties": [] + } + ] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load5{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[5m]))", + "interval": "", + "legendFormat": "Load[5m]", + "refId": "A" + }, + { + "expr": "sum(node_load1{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[1m]))", + "interval": "", + "legendFormat": "Load[1m]", + "refId": "B" + }, + { + "expr": "sum(node_load15{instance=~\"$instance\"} OR avg_over_time(windows_system_processor_queue_length{instance=~\"$instance\"}[15m]))", + "interval": "", + "legendFormat": "Load[15m]", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes{instance=~\"$instance\"} OR windows_os_physical_memory_free_bytes{instance=~\"$instance\"}) / sum(node_memory_MemTotal_bytes{instance=~\"$instance\"} OR windows_cs_physical_memory_bytes{instance=~\"$instance\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - (sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_free_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\", instance=~\"$instance\"} OR windows_logical_disk_size_bytes{volume!~\"(HarddiskVolume.+|[A-Z]:.+)\", instance=~\"$instance\"}))", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_read_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_logical_disk_write_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(node_network_receive_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "A" + }, + { + "expr": "(sum(rate(node_network_receive_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + }, + { + "expr": "(sum(rate(node_network_transmit_errs_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_errors_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "C" + }, + { + "expr": "(sum(rate(node_network_receive_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_received_discarded_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "D" + }, + { + "expr": "(sum(rate(node_network_transmit_drop_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_outbound_discarded{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "E" + }, + { + "expr": "(sum(rate(node_network_transmit_packets_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0)) + (sum(rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval])) OR on() vector(0))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_sent_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "A" + }, + { + "expr": "sum(rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\", instance=~\"$instance\"}[$__rate_interval]) OR rate(windows_net_packets_received_total_total{nic!~'.*isatap.*|.*VPN.*|.*Pseudo.*|.*tunneling.*', instance=~\"$instance\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "query": "label_values({__name__=~\"node_exporter_build_info|windows_exporter_build_info\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Node", + "uid": "rancher-node-1", + "version": 3 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/performance/performance-debugging.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/performance/performance-debugging.json new file mode 100644 index 000000000..454bc3939 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/performance/performance-debugging.json @@ -0,0 +1,1652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_sum[5m]))\n/\nsum by (handler_name) (rate(lasso_controller_reconcile_time_seconds_count[5m])))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Average Execution Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1390", + "format": "short", + "label": "Execution Time in Seconds", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1391", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(steve_api_request_time_sum{resource!=\"subscribe\"}[5m]))\n/\nsum by (resource, method, code) (rate(steve_api_request_time_count{resource!=\"subscribe\"}[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rancher API Average Request Times Over Last 5 Minutes (Top 20) (Subscribes Omitted)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:178", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:179", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "rate(steve_api_request_time_sum{resource=\"subscribe\"}[5m])\n/\nrate(steve_api_request_time_count{resource=\"subscribe\"}[5m])", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Subscribe Average Request Times Over Last 5 Minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:368", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:369", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,workqueue_depth)", + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Controller Work Queue Depth (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1553", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1554", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 13, + "w": 16, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method, code) (steve_api_total_requests))", + "instant": false, + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Rancher Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:290", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (id, resource, method) (steve_api_total_requests{code!=\"200\",code!=\"201\"}))", + "interval": "", + "legendFormat": "{{id}} {{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Number of Failed Rancher API Requests (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:428", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:429", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_store_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_store_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Store Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:662", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:663", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (resource, method, code) (rate(k8s_proxy_client_request_time_sum[5m]))\n/\nsum by (resource, method, code) (rate(k8s_proxy_client_request_time_count[5m])))", + "interval": "", + "legendFormat": "{{resource}} {{method}} {{code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "K8s Proxy Client Average Request Times Over Last 5 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1710", + "format": "ms", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1711", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 70 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,lasso_controller_total_cached_object)", + "interval": "", + "legendFormat": "{{kind}} {{version}} {{group}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cached Objects by GroupVersionKind (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:744", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:745", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 78 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Lasso Handler Executions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:824", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:825", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 86 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20, sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name) (\nlasso_controller_total_handler_execution{has_error=\"true\"}\n))", + "interval": "", + "legendFormat": "{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Handler Executions with Error (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 102 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,sum by (handler_name,controller_name) (\nincrease(lasso_controller_total_handler_execution{has_error=\"true\"}[2m])\n))", + "interval": "", + "legendFormat": "{{controller_name}}.{{handler_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Handler Executions Over Last 2 Minutes (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "topk(20,session_server_total_transmit_bytes)", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Data Transmitted by Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1953", + "format": "decbytes", + "label": "", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1954", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 118 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "exemplar": true, + "expr": "session_server_total_transmit_error_bytes", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors for Remote Dialer Sessions (Top 20)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2045", + "format": "ms", + "label": "Error Data", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2046", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 126 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "session_server_total_add_websocket_session - (session_server_total_remove_websocket_session or (0 * session_server_total_add_websocket_session))", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Active Connections (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 134 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_remove_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Removed Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2199", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2200", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(session_server_total_add_connections[$__rate_interval])", + "interval": "", + "legendFormat": "{{clientkey}} {{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Remote Dialer Added Connections Rate (Top 20)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2117", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2118", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rancher Performance Debugging", + "uid": "tfrfU0a7k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod-containers.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod-containers.json new file mode 100644 index 000000000..cf78a2204 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod-containers.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "CFS throttled ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "System ({{container}})", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Total ({{container}})", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "User ({{container}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}) by (container)", + "interval": "", + "legendFormat": "({{container}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Dropped ({{container}})", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Errors ({{container}})", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Errors ({{container}})", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Dropped ({{container}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Receive Total ({{container}})", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Transmit Total ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{container}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Write ({{container}})", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) by (container)", + "interval": "", + "legendFormat": "Read ({{container}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod (Containers)", + "uid": "rancher-pod-containers-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod.json new file mode 100644 index 000000000..4859eccc7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/pods/rancher-pod.json @@ -0,0 +1,636 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_user_seconds_total{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval])) OR sum(rate(windows_container_cpu_usage_seconds_usermode{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"} OR windows_container_memory_usage_commit_bytes{container!=\"POD\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\",pod=~\"$pod\",container!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "label_values(kube_pod_info{}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Pod", + "uid": "rancher-pod-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload-pods.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload-pods.json new file mode 100644 index 000000000..92c0d24a6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload-pods.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "CFS throttled ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "System ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Total ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\",container=\"\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "User ({{pod}})", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Dropped ({{pod}})", + "refId": "C" + }, + { + "expr": "(sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Errors ({{pod}})", + "refId": "D" + }, + { + "expr": "(sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Errors ({{pod}})", + "refId": "E" + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Dropped ({{pod}})", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Receive Total ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Transmit Total ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Write ({{pod}})", + "refId": "A" + }, + { + "expr": "(sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"}", + "interval": "", + "legendFormat": "Read ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload (Pods)", + "uid": "rancher-workload-pods-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload.json b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload.json new file mode 100644 index 000000000..9f5317c2f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/rancher/workloads/rancher-workload.json @@ -0,0 +1,652 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 28, + "iteration": 1618265214337, + "links": [], + "panels": [ + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_cpu_cfs_throttled_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "CFS throttled", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_cpu_system_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_kernelmode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "System", + "refId": "B" + }, + { + "expr": "sum((sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "C" + }, + { + "expr": "sum((sum(rate(container_cpu_user_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(rate(windows_container_cpu_usage_seconds_usermode{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "User", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "cpu", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(container_memory_working_set_bytes{namespace=~\"$namespace\"} OR windows_container_memory_usage_commit_bytes{namespace=~\"$namespace\"}) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + }, + { + "expr": "sum((sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Dropped", + "refId": "C" + }, + { + "expr": "sum((sum(irate(container_network_receive_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Errors", + "refId": "D" + }, + { + "expr": "sum((sum(irate(container_network_transmit_errors_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Errors", + "refId": "E" + }, + { + "expr": "sum((sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Dropped", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Receive Total", + "refId": "A" + }, + { + "expr": "sum((sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod) OR sum(irate(windows_container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Transmit Total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{{pod}}": "#3797d5" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((sum(rate(container_fs_writes_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Write", + "refId": "A" + }, + { + "expr": "sum((sum(rate(container_fs_reads_bytes_total{namespace=~\"$namespace\"}[$__rate_interval])) by (pod)) * on(pod) kube_pod_info{namespace=~\"$namespace\", created_by_kind=\"$kind\", created_by_name=\"$workload\"})", + "interval": "", + "legendFormat": "Read", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "query": "query_result(kube_pod_info{namespace!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*namespace=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "kind", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_kind=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "query": "query_result(kube_pod_info{namespace=\"$namespace\", created_by_kind=\"$kind\", created_by_name!=\"\"} * on(pod) group_right(namespace, created_by_kind, created_by_name) count({__name__=~\"container_.*|windows_container_.*\", pod!=\"\"}) by (pod))", + "refresh": 2, + "regex": "/.*created_by_name=\"([^\"]*)\"/", + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rancher / Workload", + "uid": "rancher-workload-1", + "version": 8 +} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh b/charts/rancher-monitoring/106.0.0+up61.3.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh new file mode 100644 index 000000000..89431e713 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/files/upgrade/scripts/delete-workloads-with-old-labels.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e +set -x + +# node-exporter +kubectl delete daemonset -l app=prometheus-node-exporter,release=rancher-monitoring --ignore-not-found=true + +# prometheus-adapter +kubectl delete deployments -l app=prometheus-adapter,release=rancher-monitoring --ignore-not-found=true + +# kube-state-metrics +kubectl delete deployments -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true +kubectl delete statefulsets -l app.kubernetes.io/instance=rancher-monitoring,app.kubernetes.io/name=kube-state-metrics --cascade=orphan --ignore-not-found=true diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/NOTES.txt b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/NOTES.txt new file mode 100644 index 000000000..371f3ae39 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/_helpers.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/_helpers.tpl new file mode 100644 index 000000000..2e964669f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/_helpers.tpl @@ -0,0 +1,472 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{- define "monitoring_registry" -}} + {{- $temp_registry := (include "system_default_registry" .) -}} + {{- if $temp_registry -}} + {{- trimSuffix "/" $temp_registry -}} + {{- else -}} + {{- .Values.global.imageRegistry -}} + {{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.enabled" -}} +{{- if or .Values.kubelet.enabled .Values.hardenedKubelet.enabled .Values.k3sServer.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "exporter.kubelet.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kubelet +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +{{- define "rancher.serviceMonitor.selector" -}} +{{- if .Values.rancherMonitoring.selector }} +{{ .Values.rancherMonitoring.selector | toYaml }} +{{- else }} +{{- $rancherDeployment := (lookup "apps/v1" "Deployment" "cattle-system" "rancher") }} +{{- if $rancherDeployment }} +matchLabels: + app: rancher + chart: {{ index $rancherDeployment.metadata.labels "chart" }} + release: rancher +{{- end }} +{{- end }} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with -operator */}} +{{/* Adding 9 to 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- if .Values.prometheusOperator.fullnameOverride -}} +{{- .Values.prometheusOperator.fullnameOverride | trunc 35 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} +{{- end }} + +{{/* Prometheus custom resource instance name */}} +{{- define "kube-prometheus-stack.prometheus.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-prometheus" }} +{{- end }} +{{- end }} + +{{/* Prometheus apiVersion for networkpolicy */}} +{{- define "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} + +{{/* Alertmanager custom resource instance name */}} +{{- define "kube-prometheus-stack.alertmanager.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" .) "-alertmanager" -}} +{{- end }} +{{- end }} + +{{/* ThanosRuler custom resource instance name */}} +{{/* Subtracting 1 from 26 truncation of kube-prometheus-stack.fullname */}} +{{- define "kube-prometheus-stack.thanosRuler.crname" -}} +{{- if .Values.cleanPrometheusOperatorObjectNames }} +{{- include "kube-prometheus-stack.fullname" . }} +{{- else }} +{{- print (include "kube-prometheus-stack.fullname" . | trunc 25 | trimSuffix "-") "-thanos-ruler" -}} +{{- end }} +{{- end }} + +{{/* Shortened name suffixed with thanos-ruler */}} +{{- define "kube-prometheus-stack.thanosRuler.name" -}} +{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}} +{{- end }} + + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (printf "%s-webhook" (include "kube-prometheus-stack.operator.fullname" .)) .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-prometheus") .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (print (include "kube-prometheus-stack.fullname" .) "-alertmanager") .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of thanosRuler service account to use */}} +{{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}} +{{- if .Values.thanosRuler.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.thanosRuler.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Allow kube-state-metrics job name to be overridden +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.name" -}} + {{- if index .Values "kube-state-metrics" "nameOverride" -}} + {{- index .Values "kube-state-metrics" "nameOverride" -}} + {{- else -}} + {{- print "kube-state-metrics" -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Sets default scrape limits for servicemonitor */}} +{{- define "servicemonitor.scrapeLimits" -}} +{{- with .sampleLimit }} +sampleLimit: {{ . }} +{{- end }} +{{- with .targetLimit }} +targetLimit: {{ . }} +{{- end }} +{{- with .labelLimit }} +labelLimit: {{ . }} +{{- end }} +{{- with .labelNameLengthLimit }} +labelNameLengthLimit: {{ . }} +{{- end }} +{{- with .labelValueLengthLimit }} +labelValueLengthLimit: {{ . }} +{{- end }} +{{- end -}} + +{{/* +To help compatibility with other charts which use global.imagePullSecrets. +Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style). +global: + imagePullSecrets: + - name: pullSecret1 + - name: pullSecret2 + +or + +global: + imagePullSecrets: + - pullSecret1 + - pullSecret2 +*/}} +{{- define "kube-prometheus-stack.imagePullSecrets" -}} +{{- range .Values.global.imagePullSecrets }} + {{- if eq (typeOf .) "map[string]interface {}" }} +- {{ toYaml . | trim }} + {{- else }} +- name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{- define "kube-prometheus-stack.operator.admission-webhook.dnsNames" }} +{{- $fullname := include "kube-prometheus-stack.operator.fullname" . }} +{{- $namespace := include "kube-prometheus-stack.namespace" . }} +{{- $fullname }} +{{ $fullname }}.{{ $namespace }}.svc +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +{{ $fullname }}-webhook +{{ $fullname }}-webhook.{{ $namespace }}.svc +{{- end }} +{{- end }} + +{{- define "rke2-ingress-nginx.namespace" -}} + {{- if .Values.rke2IngressNginx.namespaceOverride -}} + {{- .Values.rke2IngressNginx.namespaceOverride -}} + {{- else -}} + {{- print "kube-system" -}} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/alertmanager.yaml new file mode 100644 index 000000000..fdc9a8af6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,195 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.annotations }} + annotations: +{{ toYaml .Values.alertmanager.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.alertmanager.alertmanagerSpec.image.registry }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} + version: {{ default .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.version }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.alertmanager.alertmanagerSpec.automountServiceAccountToken }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ .Values.namespaceOverride }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} + {{- with .Values.alertmanager.enableFeatures }} + enableFeatures: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} + alertmanagerConfigSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4) . }} +{{ else }} + alertmanagerConfigSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} + alertmanagerConfigNamespaceSelector: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4) . }} +{{ else }} + alertmanagerConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.web }} + web: +{{ toYaml .Values.alertmanager.alertmanagerSpec.web | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }} + alertmanagerConfiguration: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }} + alertmanagerConfigMatcherStrategy: +{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ tpl (toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4) . }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.alertmanager.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.initContainers }} + initContainers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} + clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} + clusterGossipInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterGossipInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} + clusterPeerTimeout: {{ .Values.alertmanager.alertmanagerSpec.clusterPeerTimeout }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} + clusterPushpullInterval: {{ .Values.alertmanager.alertmanagerSpec.clusterPushpullInterval }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} + forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.minReadySeconds }} + minReadySeconds: {{ .Values.alertmanager.alertmanagerSpec.minReadySeconds }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.alertmanager.alertmanagerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/extrasecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/extrasecret.yaml new file mode 100644 index 000000000..ecd8f4702 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingress.yaml new file mode 100644 index 000000000..be9f5aa27 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingress.yaml @@ -0,0 +1,78 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $backendServiceName := .Values.alertmanager.ingress.serviceName | default (printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager") }} +{{- $servicePort := .Values.alertmanager.ingress.servicePort | default .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.alertmanager.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.alertmanager.ingress.ingressClassName }} + ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ | quote }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $backendServiceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $backendServiceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingressperreplica.yaml new file mode 100644 index 000000000..b2e00a416 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/podDisruptionBudget.yaml new file mode 100644 index 000000000..b18340312 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-role.yaml new file mode 100644 index 000000000..e8da52e0f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-role.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-rolebinding.yaml new file mode 100644 index 000000000..71a8ec41d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp.yaml new file mode 100644 index 000000000..5a940afab --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/psp.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/secret.yaml new file mode 100644 index 000000000..d4c397fa4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/secret.yaml @@ -0,0 +1,37 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{/* This file is applied when the operation is helm install and the target secret does not exist. */}} +{{- $secretName := (printf "alertmanager-%s" (include "kube-prometheus-stack.alertmanager.crname" .)) }} +{{- if or (not (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName)) (eq .Values.alertmanager.secret.recreateIfExists true) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "3" + "helm.sh/resource-policy": keep +{{- if .Values.alertmanager.secret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} +{{- if .Values.alertmanager.stringConfig }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }} +{{- else if eq (typeOf .Values.alertmanager.config) "string" }} + alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- end }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end }} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/service.yaml new file mode 100644 index 000000000..858c83a4c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/service.yaml @@ -0,0 +1,72 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.alertmanager.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + - name: reloader-web + {{- if semverCompare ">=1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: 8080 + targetPort: reloader-web +{{- if .Values.alertmanager.service.additionalPorts }} +{{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" . }} +{{- if .Values.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.alertmanager.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.alertmanager.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.alertmanager.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.alertmanager.service.type }}" +{{- if .Values.alertmanager.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.alertmanager.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.alertmanager.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceaccount.yaml new file mode 100644 index 000000000..745ced8bd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.alertmanager.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2}} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/servicemonitor.yaml new file mode 100644 index 000000000..ffba880ae --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,84 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.alertmanager.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + enableHttp2: {{ .Values.alertmanager.serviceMonitor.enableHttp2 }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.alertmanager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.scheme }} + scheme: {{ .Values.alertmanager.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.alertmanager.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.alertmanager.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.alertmanager.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.alertmanager.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.alertmanager.serviceMonitor.interval .interval }} + interval: {{ default $.Values.alertmanager.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.alertmanager.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.alertmanager.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.alertmanager.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.alertmanager.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.alertmanager.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.alertmanager.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.alertmanager.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceperreplica.yaml new file mode 100644 index 000000000..75a13bdf9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app.kubernetes.io/name: alertmanager + alertmanager: {{ template "kube-prometheus-stack.alertmanager.crname" $ }} + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.alertmanager.crname" $ }}-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/service.yaml new file mode 100644 index 000000000..5dedc369d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.coreDns.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.coreDns.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.coreDns.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: {{ .Values.coreDns.serviceMonitor.port }} + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/servicemonitor.yaml new file mode 100644 index 000000000..dc15a0693 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.coreDns.enabled .Values.coreDns.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + {{- with .Values.coreDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.coreDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.coreDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.coreDns.serviceMonitor.port }} + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + {{- if .Values.coreDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.coreDns.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName }} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100644 index 000000000..66e777632 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.kubeApiServer.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: default + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver + {{- with .Values.kubeApiServer.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }} + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }} + {{- end }} + port: https + scheme: https + metricRelabelings: + {{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100644 index 000000000..6a6afa641 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/service.yaml new file mode 100644 index 000000000..0a901c4ac --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.kubeControllerManager.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.kubeControllerManager.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.kubeControllerManager.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100644 index 000000000..7ed3baa65 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + {{- with .Values.kubeControllerManager.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeControllerManager.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeControllerManager.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeControllerManager.serviceMonitor.port }} + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} + metricRelabelings: + {{- if.Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/service.yaml new file mode 100644 index 000000000..478f41940 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.kubeDns.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.kubeDns.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.kubeDns.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100644 index 000000000..9fa41b575 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.kubeDns.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + {{- with .Values.kubeDns.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeDns.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeDns.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeDns.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeDns.serviceMonitor.proxyUrl}} + {{- end }} + metricRelabelings: + {{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + {{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/endpoints.yaml new file mode 100644 index 000000000..e36644757 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/service.yaml new file mode 100644 index 000000000..a62059aa4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.kubeEtcd.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.kubeEtcd.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.kubeEtcd.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: {{ .Values.kubeEtcd.serviceMonitor.port }} + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100644 index 000000000..26fdbdbed --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,75 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + {{- with .Values.kubeEtcd.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeEtcd.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }} + selector: + {{- if .Values.kubeEtcd.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeEtcd.serviceMonitor.port }} + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeEtcd.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeEtcd.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} + metricRelabelings: + {{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/endpoints.yaml new file mode 100644 index 000000000..8613e6242 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/service.yaml new file mode 100644 index 000000000..672f5492b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.kubeProxy.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.kubeProxy.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.kubeProxy.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: {{ .Values.kubeProxy.serviceMonitor.port }} + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100644 index 000000000..24b0ab200 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + {{- with .Values.kubeProxy.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeProxy.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeProxy.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeProxy.serviceMonitor.port }} + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeProxy.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} + metricRelabelings: + {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100644 index 000000000..6236b42f1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/service.yaml new file mode 100644 index 000000000..8663d79f0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.service.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + {{- if .Values.kubeScheduler.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.kubeScheduler.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.kubeScheduler.service.ipDualStack.ipFamilyPolicy }} + {{- end }} + ports: + - name: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} + protocol: TCP + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100644 index 000000000..b17c4f1d4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.serviceMonitor.enabled .Values.kubernetesServiceMonitors.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: kube-system + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + {{- with .Values.kubeScheduler.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.kubeScheduler.serviceMonitor.jobLabel }} + {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }} + selector: + {{- if .Values.kubeScheduler.serviceMonitor.selector }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.selector | nindent 4) . }} + {{- else }} + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + {{- end }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: {{ .Values.kubeScheduler.serviceMonitor.port }} + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} + metricRelabelings: + {{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-state-metrics/validate.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-state-metrics/validate.yaml new file mode 100644 index 000000000..9211b3d77 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kube-state-metrics/validate.yaml @@ -0,0 +1,7 @@ +{{- if .Values.kubeStateMetrics.enabled }} +{{- if not (kindIs "invalid" .Values.kubeStateMetrics.serviceMonitor) }} +{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} +{{- fail "kubeStateMetrics.serviceMonitor.namespaceOverride was removed. Please use kube-state-metrics.namespaceOverride instead." }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kubelet/servicemonitor.yaml new file mode 100644 index 000000000..f570fbfdb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,246 @@ +{{- if (and (not .Values.kubelet.enabled) .Values.hardenedKubelet.enabled) }} +{{ required "Cannot set .Values.hardenedKubelet.enabled=true when .Values.kubelet.enabled=false" "" }} +{{- end }} +{{- if (and .Values.kubelet.enabled .Values.kubernetesServiceMonitors.enabled (not .Values.hardenedKubelet.enabled) (not .Values.k3sServer.enabled)) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + namespace: {{ .Values.kubelet.namespace }} + {{- else }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet + {{- with .Values.kubelet.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }} + {{- with .Values.kubelet.serviceMonitor.attachMetadata }} + attachMetadata: + {{- toYaml . | nindent 4 }} + {{- end }} + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + metricRelabelings: + {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: http-metrics + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: {{ .Values.kubelet.serviceMonitor.honorLabels }} + honorTimestamps: {{ .Values.kubelet.serviceMonitor.honorTimestamps }} +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + app.kubernetes.io/name: kubelet + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/node-exporter/validate.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/node-exporter/validate.yaml new file mode 100644 index 000000000..bdc73d616 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/exporters/node-exporter/validate.yaml @@ -0,0 +1,3 @@ +{{- if (and (not .Values.nodeExporter.enabled) .Values.hardenedNodeExporter.enabled) }} +{{ required "Cannot set .Values.hardenedNodeExporter.enabled=true when .Values.nodeExporter.enabled=false" "" }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/extra-objects.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/extra-objects.yaml new file mode 100644 index 000000000..567f7bf32 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/extra-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraManifests }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmap-dashboards.yaml new file mode 100644 index 000000000..e719009ff --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmaps-datasources.yaml new file mode 100644 index 000000000..135bad440 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,81 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: + {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} + datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: "{{ .Values.grafana.sidecar.datasources.name }}" + type: prometheus + uid: {{ .Values.grafana.sidecar.datasources.uid }} + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} + access: proxy + isDefault: {{ .Values.grafana.sidecar.datasources.isDefaultDatasource }} + jsonData: + httpMethod: {{ .Values.grafana.sidecar.datasources.httpMethod }} + timeInterval: {{ $scrapeInterval }} + {{- if .Values.grafana.sidecar.datasources.timeout }} + timeout: {{ .Values.grafana.sidecar.datasources.timeout }} + {{- end }} +{{- if .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ .Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: "{{ $.Values.grafana.sidecar.datasources.name }}-{{ . }}" + type: prometheus + uid: {{ $.Values.grafana.sidecar.datasources.uid }}-replica-{{ . }} + url: http://prometheus-{{ template "kube-prometheus-stack.prometheus.crname" $ }}-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false + jsonData: + timeInterval: {{ $scrapeInterval }} +{{- if $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations }} + exemplarTraceIdDestinations: + - datasourceUid: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.datasourceUid }} + name: {{ $.Values.grafana.sidecar.datasources.exemplarTraceIdDestinations.traceIdLabelName }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.sidecar.datasources.alertmanager.enabled }} + - name: "{{ .Values.grafana.sidecar.datasources.alertmanager.name }}" + type: alertmanager + uid: {{ .Values.grafana.sidecar.datasources.alertmanager.uid }} + {{- if .Values.grafana.sidecar.datasources.alertmanager.url }} + url: {{ .Values.grafana.sidecar.datasources.alertmanager.url }} + {{- else }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }}/{{ trimPrefix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }} + {{- end }} + access: proxy + jsonData: + handleGrafanaManagedAlerts: {{ .Values.grafana.sidecar.datasources.alertmanager.handleGrafanaManagedAlerts }} + implementation: {{ .Values.grafana.sidecar.datasources.alertmanager.implementation }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 000000000..216467bdf --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,616 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "current set of alerts stored in the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid alerts received by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "rate of successful and invalid notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "latency of notifications sent by the Alertmanager", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "service", + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100644 index 000000000..c05d6ceeb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability guarantees?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100644 index 000000000..cf8e00540 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1882 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100644 index 000000000..507d47555 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100644 index 000000000..0eeedc629 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1229 @@ +{{- /* +Generated from 'etcd' from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[$__rate_interval])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[$__rate_interval])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[$__rate_interval]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 42, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.3", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=\"$cluster\"}[$__rate_interval])))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer round trip time", + "metric": "etcd_network_peer_round_trip_time_seconds_bucket", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peer round trip time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:925", + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:926", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "etcd-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 2, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/grafana-overview.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/grafana-overview.yaml new file mode 100644 index 000000000..f8c8884d0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/grafana-overview.yaml @@ -0,0 +1,635 @@ +{{- /* +Generated from 'grafana-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "grafana-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + grafana-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3085, + "iteration": 1631554945276, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Firing Alerts", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": { + + }, + "textMode": "auto" + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dashboards", + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "displayMode": "auto" + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.3", + "targets": [ + { + "expr": "grafana_build_info{job=~\"$job\", instance=~\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Build Info", + "transformations": [ + { + "id": "labelsToFields", + "options": { + + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "branch": true, + "container": true, + "goversion": true, + "namespace": true, + "pod": true, + "revision": true + }, + "indexByName": { + "Time": 7, + "Value": 11, + "branch": 4, + "container": 8, + "edition": 2, + "goversion": 6, + "instance": 1, + "job": 0, + "namespace": 9, + "pod": 10, + "revision": 5, + "version": 3 + }, + "renameByName": { + + } + } + } + ], + "type": "table" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ", + "interval": "", + "legendFormat": "{{`{{`}}status_code{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:157", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:158", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.1.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1", + "interval": "", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "$$hashKey": "object:210", + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:211", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 30, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": [ + "default/grafana" + ], + "value": [ + "default/grafana" + ] + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "job", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, job)", + "refId": "Billing Admin-job-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(grafana_build_info, instance)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "instance", + "options": [ + + ], + "query": { + "query": "label_values(grafana_build_info, instance)", + "refId": "Billing Admin-instance-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Grafana Overview", + "uid": "6be0s85Mk", + "version": 2 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100644 index 000000000..7ecca76f2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1534 @@ +{{- /* +Generated from 'k8s-coredns' from ../files/dashboards/k8s-coredns.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.", + "editable": true, + "gnetId": 12539, + "graphTooltip": 0, + "iteration": 1603798405693, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "CoreDNS.io", + "type": "link", + "url": "https://coredns.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (zone)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{zone}}"}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=\"coredns\",instance=~\"$instance\"}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (rcode)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{rcode}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"coredns\",instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=\"coredns\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{"{{proto}}"}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{job=\"coredns\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=\"coredns\",instance=~\"$instance\"}) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{"{{type}}"}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hits:{{"{{type}}"}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{job=\"coredns\",instance=~\"$instance\"}[5m])) by (type)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "definition": "label_values(up{job=\"coredns\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"coredns\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 2 + } +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100644 index 000000000..5c1193274 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,3088 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Namespace", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml new file mode 100644 index 000000000..75e158420 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-multicluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-multicluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-multicluster.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":3,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"cluster:node_cpu:ratio_rate5m","instant":true}],"title":"CPU Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","instant":true}],"title":"CPU Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","instant":true}],"title":"CPU Limits Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":12,"y":0},"id":4,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","instant":true}],"title":"Memory Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":16,"y":0},"id":5,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","instant":true}],"title":"Memory Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":20,"y":0},"id":6,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","instant":true}],"title":"Memory Limits Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"showPoints":"never"}}},"gridPos":{"h":7,"w":24,"x":0,"y":1},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","legendFormat":"__auto"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Cluster"},"properties":[{"id":"links","value":[{"title":"Drill down","url":"/d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster?${datasource:queryparam}&var-cluster=${__data.fields.Cluster}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":2},"id":8,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true}],"title":"CPU Quota","transformations":[{"id":"joinByField","options":{"byField":"cluster","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"cluster":5},"renameByName":{"Value #A":"CPU Usage","Value #B":"CPU Requests","Value #C":"CPU Requests %","Value #D":"CPU Limits","Value #E":"CPU Limits %","cluster":"Cluster"}}}],"type":"table"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":24,"x":0,"y":3},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","legendFormat":"__auto"}],"title":"Memory Usage (w/o cache)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Cluster"},"properties":[{"id":"links","value":[{"title":"Drill down","url":"/d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster?${datasource:queryparam}&var-cluster=${__data.fields.Cluster}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":4},"id":10,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true}],"title":"Memory Requests by Cluster","transformations":[{"id":"joinByField","options":{"byField":"cluster","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"cluster":5},"renameByName":{"Value #A":"Memory Usage","Value #B":"Memory Requests","Value #C":"Memory Requests %","Value #D":"Memory Limits","Value #E":"Memory Limits %","cluster":"Cluster"}}}],"type":"table"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100644 index 000000000..896b0b2d9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2797 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100644 index 000000000..ab9c76efe --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,1026 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100644 index 000000000..0a6da86c1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,2469 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (WSS)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage (WSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml new file mode 100644 index 000000000..6def97a79 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-cluster.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-cluster' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-cluster.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":3,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode=\"idle\"}[1m]))","instant":true}],"title":"CPU Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":4,"y":0},"id":2,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","instant":true}],"title":"CPU Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":8,"y":0},"id":3,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})","instant":true}],"title":"CPU Limits Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":12,"y":0},"id":4,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","instant":true}],"title":"Memory Utilisation","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":16,"y":0},"id":5,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","instant":true}],"title":"Memory Requests Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"gridPos":{"h":3,"w":4,"x":20,"y":0},"id":6,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})","instant":true}],"title":"Memory Limits Commitment","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}}},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","legendFormat":"__auto"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Namespace"},"properties":[{"id":"links","value":[{"title":"Drill down to pods","url":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":8,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true}],"title":"CPU Quota","transformations":[{"id":"joinByField","options":{"byField":"namespace","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"namespace":5},"renameByName":{"Value #A":"CPU Usage","Value #B":"CPU Requests","Value #C":"CPU Requests %","Value #D":"CPU Limits","Value #E":"CPU Limits %","namespace":"Namespace"}}}],"type":"table"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"decbytes"}},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","legendFormat":"__auto"}],"title":"Memory Usage (Private Working Set)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Memory Usage"},"properties":[{"id":"unit","value":"decbytes"}]},{"matcher":{"id":"byName","options":"Memory Requests"},"properties":[{"id":"unit","value":"decbytes"}]},{"matcher":{"id":"byName","options":"Memory Limits"},"properties":[{"id":"unit","value":"decbytes"}]},{"matcher":{"id":"byName","options":"Namespace"},"properties":[{"id":"links","value":[{"title":"Drill down to pods","url":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":10,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true}],"title":"Memory Requests by Namespace","transformations":[{"id":"joinByField","options":{"byField":"namespace","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"namespace":5},"renameByName":{"Value #A":"Memory Usage","Value #B":"Memory Requests","Value #C":"Memory Requests %","Value #D":"Memory Limits","Value #E":"Memory Limits %","namespace":"Namespace"}}}],"type":"table"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"label":"cluster","name":"cluster","query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Cluster(Windows)","uid":"4d08557fd9391b100730f2494bccac68"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml new file mode 100644 index 000000000..2dfd58bed --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-namespace.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-namespace' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-namespace.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}}},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","legendFormat":"__auto"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Pod"},"properties":[{"id":"links","value":[{"title":"Drill down to pods","url":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true}],"title":"CPU Quota","transformations":[{"id":"joinByField","options":{"byField":"pod","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"pod":5},"renameByName":{"Value #A":"CPU Usage","Value #B":"CPU Requests","Value #C":"CPU Requests %","Value #D":"CPU Limits","Value #E":"CPU Limits %","pod":"Pod"}}}],"type":"table"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"decbytes"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","legendFormat":"__auto"}],"title":"Memory Usage (Private Working Set)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Pod"},"properties":[{"id":"links","value":[{"title":"Drill down to pods","url":"/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":4,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true}],"title":"Memory Quota","transformations":[{"id":"joinByField","options":{"byField":"pod","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"pod":5},"renameByName":{"Value #A":"Memory Usage","Value #B":"Memory Requests","Value #C":"Memory Requests %","Value #D":"Memory Limits","Value #E":"Memory Limits %","pod":"Pod"}}}],"type":"table"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"label":"cluster","name":"cluster","query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"label":"namespace","name":"namespace","query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Namespace(Windows)","uid":"490b402361724ab1d4c45666c1fa9b6f"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml new file mode 100644 index 000000000..4417b31a6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-windows-pod.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-resources-windows-pod' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-windows-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-windows-pod.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true}}},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","legendFormat":"__auto"}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]},{"matcher":{"id":"byName","options":"Namespace"},"properties":[{"id":"links","value":[{"title":"Drill down to pods","url":"/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"}]}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":2,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true}],"title":"CPU Quota","transformations":[{"id":"joinByField","options":{"byField":"container","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"container":5},"renameByName":{"Value #A":"CPU Usage","Value #B":"CPU Requests","Value #C":"CPU Requests %","Value #D":"CPU Limits","Value #E":"CPU Limits %","container":"Container"}}}],"type":"table"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"decbytes"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","legendFormat":"__auto"}],"title":"Memory Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/%/"},"properties":[{"id":"unit","value":"percentunit"}]}]},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":4,"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true}],"title":"Memory Quota","transformations":[{"id":"joinByField","options":{"byField":"container","mode":"outer"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 3":true,"Time 4":true,"Time 5":true},"indexByName":{"Time 1":0,"Time 2":1,"Time 3":2,"Time 4":3,"Time 5":4,"Value #A":6,"Value #B":7,"Value #C":8,"Value #D":9,"Value #E":10,"container":5},"renameByName":{"Value #A":"Memory Usage","Value #B":"Memory Requests","Value #C":"Memory Requests %","Value #D":"Memory Limits","Value #E":"Memory Limits %","container":"Container"}}}],"type":"table"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","legendFormat":"Received : {{ container }}"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[1m])))","legendFormat":"Transmitted : {{ container }}"}],"title":"Network I/O","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"label":"cluster","name":"cluster","query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"label":"namespace","name":"namespace","query":"label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)","refresh":2,"sort":1,"type":"query"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"label":"pod","name":"pod","query":"label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)","refresh":2,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / Compute Resources / Pod(Windows)","uid":"40597a704a610e936dc6ed374a7ce023"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100644 index 000000000..a1bd68b80 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2024 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Pod", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100644 index 000000000..09257b911 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2189 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Network Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Container Bandwidth by Workload", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rate of Packets Dropped", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml new file mode 100644 index 000000000..0c4a2ca99 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-cluster-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-cluster-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-cluster-rsrc-use.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":24,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))","legendFormat":"{{instance}}"}],"title":"CPU Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":12,"x":0,"y":7},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}","legendFormat":"{{instance}}"}],"title":"Memory Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":12,"x":12,"y":7},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}","legendFormat":"{{instance}}"}],"title":"Memory Saturation (Swap I/O Pages)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})","legendFormat":"{{instance}}"}],"title":"Disk IO Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"Bps"}},"gridPos":{"h":7,"w":12,"x":0,"y":21},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}","legendFormat":"{{instance}}"}],"title":"Net Utilisation (Transmitted)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"Bps"}},"gridPos":{"h":7,"w":12,"x":12,"y":21},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}","legendFormat":"{{instance}}"}],"title":"Net Utilisation (Dropped)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})","legendFormat":"{{instance}}"}],"title":"Disk Capacity","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"label":"cluster","name":"cluster","query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Cluster(Windows)","uid":"53a43377ec9aaf2ff64dfc7a1f539334"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml new file mode 100644 index 000000000..890cef3a8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/k8s-windows-node-rsrc-use.yaml @@ -0,0 +1,24 @@ +{{- /* +Generated from 'k8s-windows-node-rsrc-use' from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.windowsMonitoring.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-windows-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-windows-node-rsrc-use.json: |- + {{`{"editable":`}}{{ .Values.grafana.defaultDashboardsEditable }}{{`,"panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":12,"x":0,"y":0},"id":1,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Utilisation"}],"title":"CPU Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":12,"x":12,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))","legendFormat":"{{core}}"}],"title":"CPU Usage Per Core","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":8,"x":0,"y":7},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Memory"}],"title":"Memory Utilisation %","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":8,"y":7},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(\n windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n - windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}\n)\n","legendFormat":"memory used"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})","legendFormat":"memory cached"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"})","legendFormat":"memory free"}],"title":"Memory Usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":7},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Swap IO"}],"title":"Memory Saturation (Swap I/O) Pages","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":12,"x":0,"y":14},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Utilisation"}],"title":"Disk IO Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"},"overrides":[{"matcher":{"id":"byRegexp","options":"/io time/"},"properties":[{"id":"unit","value":"ms"}]}]},"gridPos":{"h":7,"w":12,"x":12,"y":14},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","legendFormat":"read"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","legendFormat":"written"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"windows-exporter\", instance=\"$instance\"}[2m]))","legendFormat":"io time"}],"title":"Disk IO","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"percentunit"}},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"{{volume}}"}],"title":"Disk Utilisation","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"Bps"}},"gridPos":{"h":7,"w":12,"x":0,"y":28},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Utilisation"}],"title":"Net Utilisation (Transmitted)","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"Bps"}},"gridPos":{"h":7,"w":12,"x":12,"y":28},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.0.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}","legendFormat":"Saturation"}],"title":"Net Saturation (Dropped)","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":`}}{{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}{{`,"label":"cluster","name":"cluster","query":"label_values(up{job=\"windows-exporter\"}, cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"label":"instance","multi":true,"name":"instance","query":"label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)","refresh":2,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone": "`}}{{ .Values.grafana.defaultDashboardsTimezone }}{{`","title":"Kubernetes / USE Method / Node(Windows)","uid":"96e7484b0bb53b74fbc2bcb7723cd40b"}`}} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100644 index 000000000..bdbf1c219 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2256 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubelet.enabled" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Containers", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100644 index 000000000..92882d7db --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1464 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100644 index 000000000..5abea9777 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1736 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100644 index 000000000..a4bf29fd1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,1063 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Cluster", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100644 index 000000000..9c1a8fe59 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,1089 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page Faults", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "refId": "A" + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / USE Method / Node", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml new file mode 100644 index 000000000..562802fa0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes-darwin.yaml @@ -0,0 +1,1073 @@ +{{- /* +Generated from 'nodes-darwin' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.darwin.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes-darwin" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes-darwin.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Physical Memory", + "refId": "A" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory Used", + "refId": "B" + }, + { + "expr": "(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "App Memory", + "refId": "C" + }, + { + "expr": "node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Wired Memory", + "refId": "D" + }, + { + "expr": "node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compressed", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / MacOS", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100644 index 000000000..08e567b2f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,1066 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled (and (or .Values.nodeExporter.enabled .Values.nodeExporter.forceDeployDashboards) .Values.nodeExporter.operatingSystems.linux.enabled) }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + + }, + "id": 5, + "span": 3, + "targets": [ + { + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Memory Usage", + "transparent": false, + "type": "gauge" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 93 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "custom.width", + "value": 72 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + + }, + "id": 7, + "span": 6, + "targets": [ + { + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + }, + { + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "" + } + ], + "title": "Disk Space Usage", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ + + ], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": { + + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + + }, + "indexByName": { + + }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + + }, + "sort": [ + { + "field": "Mounted on" + } + ] + } + } + ], + "transparent": false, + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network received (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Network transmitted (bits/s)", + "fill": 0, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "node-exporter-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Node Exporter / Nodes", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100644 index 000000000..0e12e0a7b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,587 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100644 index 000000000..b174822a5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1228 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100644 index 000000000..5fc57e1b2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1674 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100644 index 000000000..b1f9d7333 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1235 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "60s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "s" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded body size limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Prometheus / Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100644 index 000000000..69b8abcd6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1276 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 11, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 12, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", cluster=\"$cluster\", job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100644 index 000000000..4d439c898 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + {{- if .Values.k3sServer.enabled }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", metrics_path=\"/metrics\"})", + {{- else }} + "expr": "sum(up{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + {{- end }} + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100644 index 000000000..8784c7116 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1438 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/b5b59bc0b45508b85647eb7a84b96dc167be15f1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }} + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{job=\"{{ include "exporter.kubelet.jobName" . }}\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "{{ .Values.grafana.defaultDashboardsTimezone }}", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/namespaces.yaml new file mode 100644 index 000000000..39ed210ed --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/_prometheus-operator.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/_prometheus-operator.tpl new file mode 100644 index 000000000..6ae9dc72e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/_prometheus-operator.tpl @@ -0,0 +1,7 @@ +{{/* Generate basic labels for prometheus-operator */}} +{{- define "kube-prometheus-stack.prometheus-operator.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app: {{ template "kube-prometheus-stack.name" . }}-operator +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl new file mode 100644 index 000000000..f419caf54 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/_prometheus-operator-webhook.tpl @@ -0,0 +1,6 @@ +{{/* Generate basic labels for prometheus-operator-webhook */}} +{{- define "kube-prometheus-stack.prometheus-operator-webhook.labels" }} +{{- include "kube-prometheus-stack.labels" . }} +app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator +app.kubernetes.io/component: prometheus-operator-webhook +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml new file mode 100644 index 000000000..054eac4a7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/deployment.yaml @@ -0,0 +1,143 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.annotations | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.prometheusOperator.admissionWebhooks.deployment.replicas }} + revisionHistoryLimit: {{ .Values.prometheusOperator.admissionWebhooks.deployment.revisionHistoryLimit }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podLabels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.deployment.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: prometheus-operator-admission-webhook + {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.admissionWebhooks.deployment.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + - --log-format={{ .Values.prometheusOperator.admissionWebhooks.deployment.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + - --log-level={{ .Values.prometheusOperator.admissionWebhooks.deployment.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - "--web.enable-tls=true" + - "--web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }}" + - "--web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }}" + - "--web.listen-address=:{{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }}" + - "--web.tls-min-version={{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.tlsMinVersion }}" + ports: + - containerPort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "https" "http" }} + scheme: {{ .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled | ternary "HTTPS" "HTTP" }} + initialDelaySeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.prometheusOperator.admissionWebhooks.deployment.livenessProbe.failureThreshold }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.containerSecurityContext | indent 12 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + volumeMounts: + - name: tls-secret + mountPath: /cert + readOnly: true + volumes: + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }}-webhook + automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.deployment.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml new file mode 100644 index 000000000..04458b967 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget -}} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.podDisruptionBudget | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/service.yaml new file mode 100644 index 000000000..6de9cbb71 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/service.yaml @@ -0,0 +1,62 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-webhook + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.labels }} +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.deployment.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.admissionWebhooks.deployment.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.admissionWebhooks.deployment.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.admissionWebhooks.deployment.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.admissionWebhooks.deployment.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.admissionWebhooks.deployment.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator-webhook + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.admissionWebhooks.deployment.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml new file mode 100644 index 000000000..55511da36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/deployment/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.deployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.deployment.serviceAccount.automountServiceAccountToken }} +metadata: + name: {{ template "kube-prometheus-stack.operator.admissionWebhooks.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml new file mode 100644 index 000000000..f7543b0f1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-createSecret.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml new file mode 100644 index 000000000..4e3b0d922 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/ciliumnetworkpolicy-patchWebhook.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + ## Ensure this is run before the job + helm.sh/hook-weight: "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + endpointSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100644 index 000000000..169549035 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.global.rbac.pspEnabled }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100644 index 000000000..4cf1335b2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100644 index 000000000..baed83db4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,73 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | replace "\n" "," }} + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100644 index 000000000..5639cc9e8 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,74 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- $registry := include "monitoring_registry" . | default .Values.prometheusOperator.admissionWebhooks.patch.image.registry -}} + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ $registry }}/{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml new file mode 100644 index 000000000..864deb52a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-createSecret.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml new file mode 100644 index 000000000..076c46700 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/networkpolicy-patchWebhook.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + ## Ensure this is run before the job + "helm.sh/hook-weight": "-5" + {{- with .Values.prometheusOperator.admissionWebhooks.patch.annotations }} + {{ toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 6 }} + {{- end }} + egress: + - {} + policyTypes: + - Egress +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 000000000..92c624001 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,47 @@ +{{- if and (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +{{- if .Values.global.rbac.pspAnnotations }} +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" . | nindent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100644 index 000000000..f15abf439 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100644 index 000000000..30bde920b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100644 index 000000000..8dab40c60 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.prometheusOperator.admissionWebhooks.patch.serviceAccount.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100644 index 000000000..91d96b384 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,81 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.objectSelector }} + objectSelector: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100644 index 000000000..f21a9a72b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,81 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission + {{- include "kube-prometheus-stack.prometheus-operator-webhook.labels" $ | nindent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if eq .Values.prometheusOperator.admissionWebhooks.failurePolicy "IgnoreOnInstallOnly" }} + failurePolicy: {{ .Release.IsInstall | ternary "Ignore" "Fail" }} + {{- else if .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: Fail + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }}{{ if .Values.prometheusOperator.admissionWebhooks.deployment.enabled }}-webhook{{ end }} + path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} + timeoutSeconds: {{ .Values.prometheusOperator.admissionWebhooks.timeoutSeconds }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector }} + namespaceSelector: + {{- with (omit .Values.prometheusOperator.admissionWebhooks.namespaceSelector "matchExpressions") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if or .Values.prometheusOperator.denyNamespaces .Values.prometheusOperator.namespaces .Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions }} + matchExpressions: + {{- with (.Values.prometheusOperator.admissionWebhooks.namespaceSelector.matchExpressions) }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- range $namespace := mustUniq .Values.prometheusOperator.denyNamespaces }} + - {{ $namespace }} + {{- end }} + {{- else if and .Values.prometheusOperator.namespaces .Values.prometheusOperator.namespaces.additional }} + - key: kubernetes.io/metadata.name + operator: In + values: + {{- if and .Values.prometheusOperator.namespaces.releaseNamespace (default .Values.prometheusOperator.namespaces.releaseNamespace true) }} + {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} + - {{ $namespace }} + {{- end }} + {{- range $namespace := mustUniq .Values.prometheusOperator.namespaces.additional }} + - {{ $namespace }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.objectSelector }} + objectSelector: + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/aggregate-clusterroles.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/aggregate-clusterroles.yaml new file mode 100644 index 000000000..0c52000d6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/aggregate-clusterroles.yaml @@ -0,0 +1,29 @@ +{{/* This file is based on https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/rbac-crd.md */}} +{{- if and .Values.global.rbac.create .Values.global.rbac.createAggregateClusterRoles }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-crd-view + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: ["monitoring.coreos.com"] + resources: ["alertmanagers", "alertmanagerconfigs", "podmonitors", "probes", "prometheuses", "prometheusagents", "prometheusrules", "scrapeconfigs", "servicemonitors"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-crd-edit + labels: + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: ["monitoring.coreos.com"] + resources: ["alertmanagers", "alertmanagerconfigs", "podmonitors", "probes", "prometheuses", "prometheusagents", "prometheusrules", "scrapeconfigs", "servicemonitors"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/certmanager.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 000000000..cb27e49f4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,55 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + {{- include "kube-prometheus-stack.operator.admission-webhook.dnsNames" . | splitList "\n" | toYaml | nindent 4 }} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/ciliumnetworkpolicy.yaml new file mode 100644 index 000000000..07e2e9996 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/ciliumnetworkpolicy.yaml @@ -0,0 +1,40 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + endpointSelector: + matchLabels: + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + app: {{ template "kube-prometheus-stack.name" . }}-operator + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- else }} + {{- include "kube-prometheus-stack.prometheus-operator.labels" $ | nindent 6 }} + {{- end }} + egress: + {{- if and .Values.prometheusOperator.networkPolicy.cilium .Values.prometheusOperator.networkPolicy.cilium.egress }} + {{ toYaml .Values.prometheusOperator.networkPolicy.cilium.egress | nindent 6 }} + {{- else }} + - toEntities: + - kube-apiserver + {{- end }} + ingress: + - toPorts: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort | quote }} + {{- else }} + - port: "8080" + {{- end }} + protocol: "TCP" + {{- if not .Values.prometheusOperator.tls.enabled }} + rules: + http: + - method: "GET" + path: "/metrics" + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrole.yaml new file mode 100644 index 000000000..fd11b69ee --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,109 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - alertmanagers/finalizers + - alertmanagers/status + - alertmanagerconfigs + - prometheuses + - prometheuses/finalizers + - prometheuses/status + - prometheusagents + - prometheusagents/finalizers + - prometheusagents/status + - thanosrulers + - thanosrulers/finalizers + - thanosrulers/status + - scrapeconfigs + - servicemonitors + - podmonitors + - probes + - prometheusrules + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - patch + - create +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get +{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }} +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100644 index 000000000..ad9e3ef6c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/deployment.yaml new file mode 100644 index 000000000..b71e7d25d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,207 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.labels }} +{{ toYaml .Values.prometheusOperator.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.annotations | indent 4 }} +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: {{ .Values.prometheusOperator.revisionHistoryLimit }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- with .Values.prometheusOperator.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- include "kube-prometheus-stack.imagePullSecrets" . | indent 8 }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- $base_registry := (include "monitoring_registry" .) }} + {{- $configReloaderRegistry := $base_registry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}} + {{- $operatorRegistry := $base_registry | default .Values.prometheusOperator.image.registry -}} + {{- $thanosRegistry := $base_registry | default .Values.prometheusOperator.thanosImage.registry -}} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} + {{- if .Values.prometheusOperator.kubeletService.selector }} + - --kubelet-selector={{ .Values.prometheusOperator.kubeletService.selector }} + {{- end }} + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ tpl (.Values.prometheusOperator.denyNamespaces | join ",") $ }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{- $namespaces := list }} + {{- if .releaseNamespace }} + {{- $namespaces = append $namespaces $namespace }} + {{- end }} + {{- if .additional }} + {{- range $ns := .additional }} + {{- $namespaces = append $namespaces (tpl $ns $) }} + {{- end }} + {{- end }} + - --namespaces={{ $namespaces | mustUniq | join "," }} + {{- end }} + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ $base_registry | default .Values.prometheusOperator.prometheusDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ $base_registry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + {{- else }} + - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }} + {{- end }} + - --config-reloader-cpu-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).cpu) | default 0 }} + - --config-reloader-cpu-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).cpu) | default 0 }} + - --config-reloader-memory-request={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).requests).memory) | default 0 }} + - --config-reloader-memory-limit={{ (((.Values.prometheusOperator.prometheusConfigReloader.resources).limits).memory) | default 0 }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.enableProbe }} + - --enable-config-reloader-probes=true + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} + - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerInstanceSelector }} + - --alertmanager-instance-selector={{ .Values.prometheusOperator.alertmanagerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerConfigNamespaces }} + - --alertmanager-config-namespaces={{ .Values.prometheusOperator.alertmanagerConfigNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} + - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.prometheusInstanceSelector }} + - --prometheus-instance-selector={{ .Values.prometheusOperator.prometheusInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} + - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} + {{- end }} + {{- if .Values.prometheusOperator.thanosRulerInstanceSelector }} + - --thanos-ruler-instance-selector={{ .Values.prometheusOperator.thanosRulerInstanceSelector }} + {{- end }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ tpl (.Values.prometheusOperator.secretFieldSelector) $ }} + {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - --web.enable-tls=true + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} + - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} + ports: + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} + name: https + {{- else }} + ports: + - containerPort: 8080 + name: http + {{- end }} + env: + {{- range $key, $value := .Values.prometheusOperator.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: +{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }} + volumeMounts: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + mountPath: /cert + readOnly: true + {{- end }} + {{- with .Values.prometheusOperator.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.prometheusOperator.tls.enabled }} + - name: tls-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + {{- end }} + {{- with .Values.prometheusOperator.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.prometheusOperator.automountServiceAccountToken }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/networkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/networkpolicy.yaml new file mode 100644 index 000000000..cfd5b0b8c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/networkpolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.prometheusOperator.networkPolicy.enabled (eq .Values.prometheusOperator.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + egress: + - {} + ingress: + - ports: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: {{ .Values.prometheusOperator.tls.internalPort }} + {{- else }} + - port: 8080 + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + {{- if .Values.prometheusOperator.networkPolicy.matchLabels }} + {{ toYaml .Values.prometheusOperator.networkPolicy.matchLabels | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100644 index 000000000..976623896 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100644 index 000000000..01f5f3d9d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.operator.fullname" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp.yaml new file mode 100644 index 000000000..0943b5f56 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/psp.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/service.yaml new file mode 100644 index 000000000..72e0788ab --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/service.yaml @@ -0,0 +1,61 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.prometheusOperator.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.prometheusOperator.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }} +{{- end }} + ports: + {{- if not .Values.prometheusOperator.tls.enabled }} + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- end }} + {{- if .Values.prometheusOperator.tls.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/serviceaccount.yaml new file mode 100644 index 000000000..4f84974f9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +automountServiceAccountToken: {{ .Values.prometheusOperator.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/servicemonitor.yaml new file mode 100644 index 000000000..cbe79e125 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +{{- with .Values.prometheusOperator.serviceMonitor.additionalLabels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }} + endpoints: + {{- if .Values.prometheusOperator.tls.enabled }} + - port: https + scheme: https + tlsConfig: + serverName: {{ template "kube-prometheus-stack.operator.fullname" . }} + ca: + secret: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} + optional: false + {{- else }} + - port: http + {{- end }} + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} + metricRelabelings: + {{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/verticalpodautoscaler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/verticalpodautoscaler.yaml new file mode 100644 index 000000000..f225d16dd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus-operator/verticalpodautoscaler.yaml @@ -0,0 +1,40 @@ +{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1") (.Values.prometheusOperator.verticalPodAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + {{- include "kube-prometheus-stack.prometheus-operator.labels" . | nindent 4 }} +spec: + {{- with .Values.prometheusOperator.verticalPodAutoscaler.recommenders }} + recommenders: + {{- toYaml . | nindent 4 }} + {{- end }} + resourcePolicy: + containerPolicies: + - containerName: {{ template "kube-prometheus-stack.name" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.controlledResources }} + controlledResources: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + controlledValues: {{ .Values.prometheusOperator.verticalPodAutoscaler.controlledValues }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed }} + maxAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.maxAllowed | nindent 8 }} + {{- end }} + {{- if .Values.prometheusOperator.verticalPodAutoscaler.minAllowed }} + minAllowed: + {{- toYaml .Values.prometheusOperator.verticalPodAutoscaler.minAllowed | nindent 8 }} + {{- end }} + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "kube-prometheus-stack.operator.fullname" . }} + {{- with .Values.prometheusOperator.verticalPodAutoscaler.updatePolicy }} + updatePolicy: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/_rules.tpl b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/_rules.tpl new file mode 100644 index 000000000..4a8213d08 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/_rules.tpl @@ -0,0 +1,44 @@ +{{- /* +Generated file. Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- define "rules.names" }} +rules: + - "alertmanager.rules" + - "config-reloaders" + - "etcd" + - "general.rules" + - "k8s.rules.container-cpu-usage-seconds-total" + - "k8s.rules.container-memory-cache" + - "k8s.rules.container-memory-rss" + - "k8s.rules.container-memory-swap" + - "k8s.rules.container-memory-working-set-bytes" + - "k8s.rules.container-resource" + - "k8s.rules.pod-owner" + - "kube-apiserver-availability.rules" + - "kube-apiserver-burnrate.rules" + - "kube-apiserver-histogram.rules" + - "kube-apiserver-slos" + - "kube-prometheus-general.rules" + - "kube-prometheus-node-recording.rules" + - "kube-scheduler.rules" + - "kube-state-metrics" + - "kubelet.rules" + - "kubernetes-apps" + - "kubernetes-resources" + - "kubernetes-storage" + - "kubernetes-system" + - "kubernetes-system-kube-proxy" + - "kubernetes-system-apiserver" + - "kubernetes-system-kubelet" + - "kubernetes-system-controller-manager" + - "kubernetes-system-scheduler" + - "node-exporter.rules" + - "node-exporter" + - "node.rules" + - "node-network" + - "prometheus-operator" + - "prometheus" + - "windows.node.rules" + - "windows.pod.rules" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100644 index 000000000..bff930981 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100644 index 000000000..2fe8fdb81 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalPrometheusRules.yaml new file mode 100644 index 000000000..cb4aabaa7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,43 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100644 index 000000000..ebdf766fd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }} + additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }} +{{- else }} + additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ciliumnetworkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ciliumnetworkpolicy.yaml new file mode 100644 index 000000000..74d61d7c1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ciliumnetworkpolicy.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "cilium") }} +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} +spec: + endpointSelector: + {{- if .Values.prometheus.networkPolicy.cilium.endpointSelector }} + {{- toYaml .Values.prometheus.networkPolicy.cilium.endpointSelector | nindent 4 }} + {{- else }} + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.egress }} + egress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.egress | nindent 4 }} + {{- end }} + {{- if and .Values.prometheus.networkPolicy.cilium .Values.prometheus.networkPolicy.cilium.ingress }} + ingress: + {{ toYaml .Values.prometheus.networkPolicy.cilium.ingress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrole.yaml new file mode 100644 index 000000000..eabdb24b9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrole.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +{{/* fix(#3338): add required rules to use node-exporter with the RBAC proxy */}} +{{- if and .Values.nodeExporter.enabled (index .Values "prometheus-node-exporter").kubeRBACProxy.enabled }} +- apiGroups: [ "" ] + resources: + - services/{{ include "prometheus-node-exporter.fullname" (index .Subcharts "prometheus-node-exporter") }} + verbs: [ "get", "list", "watch" ] +{{- end }} +{{- if and .Values.kubeStateMetrics.enabled (index .Values "kube-state-metrics").kubeRBACProxy.enabled }} +- apiGroups: [ "" ] + resources: + - services/{{ include "kube-state-metrics.fullname" (index .Subcharts "kube-state-metrics") }} + verbs: [ "get", "list", "watch" ] +{{- end }} +{{- if .Values.prometheus.additionalRulesForClusterRole }} +{{ toYaml .Values.prometheus.additionalRulesForClusterRole | indent 0 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrolebinding.yaml new file mode 100644 index 000000000..9fc4f65da --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/csi-secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/csi-secret.yaml new file mode 100644 index 000000000..e05382f63 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/extrasecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/extrasecret.yaml new file mode 100644 index 000000000..17f3478a4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingress.yaml new file mode 100644 index 000000000..d2f6af5dd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.ingress.servicePort | default .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.ingress.annotations) . | nindent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.ingress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressThanosSidecar.yaml new file mode 100644 index 000000000..3f507cfa9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-discovery" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: + {{- tpl (toYaml .Values.prometheus.thanosIngress.annotations) . | nindent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + {{- if $apiIsStable }} + {{- if .Values.prometheus.thanosIngress.ingressClassName }} + ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ tpl (toYaml .Values.prometheus.thanosIngress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 000000000..1d76d135c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,67 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} +{{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: + {{- tpl (toYaml $ingressValues.annotations) $ | nindent 8 }} + {{- end }} + spec: + {{- if $apiIsStable }} + {{- if $ingressValues.ingressClassName }} + ingressClassName: {{ $ingressValues.ingressClassName }} + {{- end }} + {{- end }} + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/networkpolicy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/networkpolicy.yaml new file mode 100644 index 000000000..1296a7906 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/networkpolicy.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.networkPolicy.enabled (eq .Values.prometheus.networkPolicy.flavor "kubernetes") }} +apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }} +kind: NetworkPolicy +metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + {{- include "kube-prometheus-stack.labels" . | nindent 4 }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + {{- if .Values.prometheus.networkPolicy.egress }} + egress: + {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.networkPolicy.ingress }} + ingress: + {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }} + {{- end }} + policyTypes: + - Egress + - Ingress + podSelector: + {{- if .Values.prometheus.networkPolicy.podSelector }} + {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }} + {{- else }} + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/nginx-config.yaml new file mode 100644 index 000000000..e4d91f9a9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/nginx-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8081; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + + rewrite ^/k8s/clusters/.*/proxy(.*) /$1 break; + + } + } + } diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podDisruptionBudget.yaml new file mode 100644 index 000000000..48f3f1f5a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podmonitors.yaml new file mode 100644 index 000000000..4e748c23b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/podmonitors.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/prometheus.yaml new file mode 100644 index 000000000..e668b40a9 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/prometheus.yaml @@ -0,0 +1,481 @@ +{{- if .Values.prometheus.enabled }} +{{- if .Values.prometheus.agentMode }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: PrometheusAgent +{{- else }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +{{- end }} +metadata: + name: {{ template "kube-prometheus-stack.prometheus.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: + automountServiceAccountToken: {{ .Values.prometheus.prometheusSpec.automountServiceAccountToken }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.alertingEndpoints .Values.alertmanager.enabled) }} + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.scheme }} + scheme: {{ .Values.alertmanager.alertmanagerSpec.scheme }} + {{- end }} + {{- if .Values.alertmanager.alertmanagerSpec.tlsConfig }} + tlsConfig: +{{ toYaml .Values.alertmanager.alertmanagerSpec.tlsConfig | indent 10 }} + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.prometheus.prometheusSpec.image.registry -}} + {{- if and .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}@sha256:{{ .Values.prometheus.prometheusSpec.image.sha }}" + {{- else if .Values.prometheus.prometheusSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}:{{ .Values.prometheus.prometheusSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.prometheus.prometheusSpec.image.repository }}" + {{- end }} + version: {{ default .Values.prometheus.prometheusSpec.image.tag .Values.prometheus.prometheusSpec.version }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalArgs }} + additionalArgs: +{{ toYaml .Values.prometheus.prometheusSpec.additionalArgs | indent 4}} +{{- end -}} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} + enableRemoteWriteReceiver: {{ .Values.prometheus.prometheusSpec.enableRemoteWriteReceiver }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else if not (or (kindIs "invalid" .Values.global.cattle.url) (kindIs "invalid" .Values.global.cattle.clusterId)) }} + externalUrl: "{{ .Values.global.cattle.url }}/k8s/clusters/{{ .Values.global.cattle.clusterId }}/api/v1/namespaces/{{ template "kube-prometheus-stack.namespace" . }}/services/http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/proxy" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} +{{- if not .Values.prometheus.agentMode }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.exemplars }} + exemplars: + {{- toYaml .Values.prometheus.prometheusSpec.exemplars | nindent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.scrapeClasses }} + scrapeClasses: + {{- tpl (toYaml . | nindent 4) $ }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.prometheusSpec.scrapeTimeout }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tsdb }} + tsdb: + {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }} + {{- end }} +{{- end }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4) . }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4) . }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeSelector }} + probeSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.probeSelectorNilUsesHelmValues }} + probeSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + probeSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.probeNamespaceSelector }} + probeNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.probeNamespaceSelector | indent 4) . }} +{{ else }} + probeNamespaceSelector: {} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} + remoteRead: +{{- if .Values.prometheus.prometheusSpec.remoteRead }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} +{{- end }} +{{- end }} +{{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} + remoteWrite: +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} +{{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4) . }} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigSelector }} + scrapeConfigSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigSelector | indent 4) . }} +{{ else if .Values.prometheus.prometheusSpec.scrapeConfigSelectorNilUsesHelmValues }} + scrapeConfigSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + scrapeConfigSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector }} + scrapeConfigNamespaceSelector: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.scrapeConfigNamespaceSelector | indent 4) . }} +{{ else }} + scrapeConfigNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4) . }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.persistentVolumeClaimRetentionPolicy }} + persistentVolumeClaimRetentionPolicy: + {{- toYaml . | nindent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ tpl (toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4) . }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} + {{- if hasKey .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret "optional" }} + optional: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.optional }} + {{- end }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret }} + additionalAlertRelabelConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigsSecret.key }} +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if not .Values.prometheus.agentMode }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}} +{{ toYaml . | indent 4 }} +{{- end }} +{{- if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).existingSecret) }} + objectStorageConfig: + key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}" +{{- else if ((.Values.prometheus.prometheusSpec.thanos.objectStorageConfig).secret) }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- end }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs }} + arbitraryFSAccessThroughSMs: +{{ toYaml .Values.prometheus.prometheusSpec.arbitraryFSAccessThroughSMs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorLabels }} + overrideHonorLabels: {{ .Values.prometheus.prometheusSpec.overrideHonorLabels }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} + overrideHonorTimestamps: {{ .Values.prometheus.prometheusSpec.overrideHonorTimestamps }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- $prometheusDefaultRulesExcludedFromEnforce := (include "rules.names" .) | fromYaml }} +{{- if not .Values.prometheus.agentMode }} + prometheusRulesExcludedFromEnforce: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - ruleNamespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + ruleName: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce }} +{{ toYaml .Values.prometheus.prometheusSpec.prometheusRulesExcludedFromEnforce | indent 4 }} +{{- end }} +{{- end }} + excludedFromEnforcement: +{{- range $prometheusDefaultRulesExcludedFromEnforce.rules }} + - group: monitoring.coreos.com + resource: prometheusrules + namespace: "{{ template "kube-prometheus-stack.namespace" $ }}" + name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} +{{- end }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.queryLogFile }} + queryLogFile: {{ .Values.prometheus.prometheusSpec.queryLogFile }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.sampleLimit }} + sampleLimit: {{ .Values.prometheus.prometheusSpec.sampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} + enforcedKeepDroppedTargets: {{ .Values.prometheus.prometheusSpec.enforcedKeepDroppedTargets }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedSampleLimit }} + enforcedSampleLimit: {{ .Values.prometheus.prometheusSpec.enforcedSampleLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedTargetLimit }} + enforcedTargetLimit: {{ .Values.prometheus.prometheusSpec.enforcedTargetLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelLimit }} + enforcedLabelLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} + enforcedLabelNameLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelNameLengthLimit }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit}} + enforcedLabelValueLengthLimit: {{ .Values.prometheus.prometheusSpec.enforcedLabelValueLengthLimit }} +{{- end }} +{{- if and (not .Values.prometheus.agentMode) .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} + allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.minReadySeconds }} + minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} + maximumStartupDurationSeconds: {{ .Values.prometheus.prometheusSpec.maximumStartupDurationSeconds }} +{{- end }} + hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }} +{{- if .Values.prometheus.prometheusSpec.hostAliases }} + hostAliases: +{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.tracingConfig }} + tracingConfig: +{{ toYaml .Values.prometheus.prometheusSpec.tracingConfig | indent 4 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.prometheus.prometheusSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrole.yaml new file mode 100644 index 000000000..872feb606 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100644 index 000000000..50e361770 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp.yaml new file mode 100644 index 000000000..b53808daa --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/psp.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Allow adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100644 index 000000000..2d432c8f3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,305 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedReload | default false) }} + - alert: AlertmanagerFailedReload + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "AlertmanagerFailedReload" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedReload" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerMembersInconsistent | default false) }} + - alert: AlertmanagerMembersInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) group_left + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: {{ dig "AlertmanagerMembersInconsistent" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerMembersInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerFailedToSendAlerts | default false) }} + - alert: AlertmanagerFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterFailedToSendAlerts | default false) }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + ignoring (reason) group_left rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: {{ dig "AlertmanagerClusterFailedToSendAlerts" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterFailedToSendAlerts" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerConfigInconsistent | default false) }} + - alert: AlertmanagerConfigInconsistent + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + count_values by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: {{ dig "AlertmanagerConfigInconsistent" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerConfigInconsistent" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterDown | default false) }} + - alert: AlertmanagerClusterDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.AlertmanagerClusterCrashlooping | default false) }} + - alert: AlertmanagerClusterCrashlooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.alertmanager | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/alertmanager/alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace,service,cluster) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: {{ dig "AlertmanagerClusterCrashlooping" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "AlertmanagerClusterCrashlooping" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.alertmanager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/config-reloaders.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/config-reloaders.yaml new file mode 100644 index 000000000..9f554c022 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/config-reloaders.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'config-reloaders' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.configReloaders }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "config-reloaders" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: config-reloaders + rules: +{{- if not (.Values.defaultRules.disabled.ConfigReloaderSidecarErrors | default false) }} + - alert: ConfigReloaderSidecarErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.configReloaders | indent 8 }} +{{- end }} + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + for: {{ dig "ConfigReloaderSidecarErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "ConfigReloaderSidecarErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.configReloaders }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/etcd.yaml new file mode 100644 index 000000000..a1d7a508f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,461 @@ +{{- /* +Generated from 'etcd' group from https://github.com/etcd-io/etcd.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: +{{- if not (.Values.defaultRules.disabled.etcdMembersDown | default false) }} + - alert: etcdMembersDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster members are down. + expr: |- + max without (endpoint) ( + sum without (instance) (up{job=~".*etcd.*"} == bool 0) + or + count without (To) ( + sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + ) + ) + > 0 + for: {{ dig "etcdMembersDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMembersDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdInsufficientMembers | default false) }} + - alert: etcdInsufficientMembers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + summary: etcd cluster has insufficient number of members. + expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2) + for: {{ dig "etcdInsufficientMembers" "for" "3m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdInsufficientMembers" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdNoLeader | default false) }} + - alert: etcdNoLeader + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + summary: etcd cluster has no leader. + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: {{ dig "etcdNoLeader" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdNoLeader" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfLeaderChanges | default false) }} + - alert: etcdHighNumberOfLeaderChanges + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + summary: etcd cluster has high number of leader changes. + expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4 + for: {{ dig "etcdHighNumberOfLeaderChanges" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfLeaderChanges" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 1 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedGRPCRequests | default false) }} + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of failed grpc requests. + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code) + > 5 + for: {{ dig "etcdHighNumberOfFailedGRPCRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedGRPCRequests" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdGRPCRequestsSlow | default false) }} + - alert: etcdGRPCRequestsSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + summary: etcd grpc requests are slow + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + > 0.15 + for: {{ dig "etcdGRPCRequestsSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdGRPCRequestsSlow" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdMemberCommunicationSlow | default false) }} + - alert: etcdMemberCommunicationSlow + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster member communication is slow. + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: {{ dig "etcdMemberCommunicationSlow" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdMemberCommunicationSlow" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighNumberOfFailedProposals | default false) }} + - alert: etcdHighNumberOfFailedProposals + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster has high number of proposal failures. + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: {{ dig "etcdHighNumberOfFailedProposals" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighNumberOfFailedProposals" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighFsyncDurations | default false) }} + - alert: etcdHighFsyncDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile fsync durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 1 + for: {{ dig "etcdHighFsyncDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighFsyncDurations" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdHighCommitDurations | default false) }} + - alert: etcdHighCommitDurations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + summary: etcd cluster 99th percentile commit durations are too high. + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: {{ dig "etcdHighCommitDurations" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdHighCommitDurations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseQuotaLowSpace | default false) }} + - alert: etcdDatabaseQuotaLowSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + summary: etcd cluster database is running full. + expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95 + for: {{ dig "etcdDatabaseQuotaLowSpace" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseQuotaLowSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdExcessiveDatabaseGrowth | default false) }} + - alert: etcdExcessiveDatabaseGrowth + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + summary: etcd cluster database growing very fast. + expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"} + for: {{ dig "etcdExcessiveDatabaseGrowth" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdExcessiveDatabaseGrowth" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.etcdDatabaseHighFragmentationRatio | default false) }} + - alert: etcdDatabaseHighFragmentationRatio + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.etcd }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.etcd | indent 8 }} +{{- end }} + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size in use on instance {{`{{`}} $labels.instance {{`}}`}} is {{`{{`}} $value | humanizePercentage {{`}}`}} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' + runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation + summary: etcd database size in use is less than 50% of the actual allocated storage. + expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600 + for: {{ dig "etcdDatabaseHighFragmentationRatio" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "etcdDatabaseHighFragmentationRatio" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.etcd }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100644 index 000000000..632422883 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,125 @@ +{{- /* +Generated from 'general.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: +{{- if not (.Values.defaultRules.disabled.TargetDown | default false) }} + - alert: TargetDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/targetdown + summary: One or more targets are unreachable. + expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10 + for: {{ dig "TargetDown" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "TargetDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.Watchdog | default false) }} + - alert: Watchdog + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. + expr: vector(1) + labels: + severity: {{ dig "Watchdog" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.InfoInhibitor | default false) }} + - alert: InfoInhibitor + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.general }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.general | indent 8 }} +{{- end }} + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: {{ dig "InfoInhibitor" "severity" "none" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.general }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml new file mode 100644 index 000000000..19aa6b4e2 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'k8s.rules.container-cpu-usage-seconds-total' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerCpuUsageSecondsTotal }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-cpu-usage-seconds-total" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_cpu_usage_seconds_total + rules: + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerCpuUsageSecondsTotal }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml new file mode 100644 index 000000000..2a08f4383 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-cache' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryCache }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-cache" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_cache + rules: + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryCache }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml new file mode 100644 index 000000000..85b23faaf --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-rss' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryRss }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-rss" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_rss + rules: + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryRss }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml new file mode 100644 index 000000000..aae26802e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-swap' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemorySwap }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-swap" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_swap + rules: + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemorySwap }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml new file mode 100644 index 000000000..cc7fbbd06 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml @@ -0,0 +1,42 @@ +{{- /* +Generated from 'k8s.rules.container-memory-working-set-bytes' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerMemoryWorkingSetBytes }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-memory-working-set-bytes" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_memory_working_set_bytes + rules: + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerMemoryWorkingSetBytes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml new file mode 100644 index 000000000..edba0c2e0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml @@ -0,0 +1,168 @@ +{{- /* +Generated from 'k8s.rules.container-resource' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sContainerResource }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.container-resource" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.container_resource + rules: + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) + group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, cluster) ( + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left() max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sContainerResource }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml new file mode 100644 index 000000000..43207a748 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml @@ -0,0 +1,107 @@ +{{- /* +Generated from 'k8s.rules.pod-owner' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8sPodOwner }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules.pod-owner" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules.pod_owner + rules: + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) group_left(owner_name) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="{{ $kubeStateMetricsJob }}"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="{{ $kubeStateMetricsJob }}", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.k8sPodOwner }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100644 index 000000000..c61bd222a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,237 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: |- + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics/cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1, + max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod, cluster) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum + {{- if .Values.defaultRules.additionalRuleLabels }} + labels: + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: job + {{- if .Values.defaultRules.additionalRuleLabels }} + {{ toYaml .Values.defaultRules.additionalRuleLabels | nindent 8 }} + {{- end }} + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100644 index 000000000..02c8e7adb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,273 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + record: code_verb:apiserver_request_total:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code:apiserver_request_total:increase30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h])) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h])) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + ) + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d) + labels: + verb: all + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:availability30d + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: code_resource:apiserver_request_total:rate5m + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverAvailability }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml new file mode 100644 index 000000000..49f4400a5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml @@ -0,0 +1,440 @@ +{{- /* +Generated from 'kube-apiserver-burnrate.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-burnrate.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-burnrate.rules + rules: + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h])) + ) + ) + + + # errors + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h])) + - + sum by (cluster) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h])) + ) + + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverBurnrate }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: apiserver_request:burnrate6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml new file mode 100644 index 000000000..5d08aa434 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'kube-apiserver-histogram.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-histogram.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-histogram.rules + rules: + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request{{ if (semverCompare ">=1.23.0-0" $kubeTargetVersion) }}_slo{{ end }}_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverHistogram }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100644 index 000000000..a83cf9060 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,159 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 6h + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "critical" .Values.customRules }} + short: 30m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 1d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 2h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPIErrorBudgetBurn | default false) }} + - alert: KubeAPIErrorBudgetBurn + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeApiserverSlos | indent 8 }} +{{- end }} + description: The API server is burning too much error budget. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapierrorbudgetburn + summary: The API server is burning too much error budget. + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: {{ dig "KubeAPIErrorBudgetBurn" "for" "3h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + long: 3d + severity: {{ dig "KubeAPIErrorBudgetBurn" "severity" "warning" .Values.customRules }} + short: 6h + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeApiserverSlos }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100644 index 000000000..fc199f11f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,49 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusGeneral }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100644 index 000000000..63f721f42 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,93 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) + record: instance:node_cpu:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) + record: instance:node_network_receive_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) + record: instance:node_network_transmit_bytes:rate:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance, cpu)) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance) + record: instance:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) + record: cluster:node_cpu:sum_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance, cpu)) + record: cluster:node_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubePrometheusNodeRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100644 index 000000000..9f8bf6022 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,135 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100644 index 000000000..7f3600fb7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,152 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeStateMetrics }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsListErrors | default false) }} + - alert: KubeStateMetricsListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricslisterrors + summary: kube-state-metrics is experiencing errors in list operations. + expr: |- + (sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_list_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsListErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsWatchErrors | default false) }} + - alert: KubeStateMetricsWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricswatcherrors + summary: kube-state-metrics is experiencing errors in watch operations. + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}",result="error"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(rate(kube_state_metrics_watch_total{job="{{ $kubeStateMetricsJob }}"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) + > 0.01 + for: {{ dig "KubeStateMetricsWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsWatchErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardingMismatch | default false) }} + - alert: KubeStateMetricsShardingMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) != 0 + for: {{ dig "KubeStateMetricsShardingMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardingMismatch" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStateMetricsShardsMissing | default false) }} + - alert: KubeStateMetricsShardsMissing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeStateMetrics | indent 8 }} +{{- end }} + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kube-state-metrics/kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - 1 + - + sum( 2 ^ max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="{{ $kubeStateMetricsJob }}"}) ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + != 0 + for: {{ dig "KubeStateMetricsShardsMissing" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStateMetricsShardsMissing" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeStateMetrics }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100644 index 000000000..8cd03baa4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kubelet.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubelet }} +{{- if (include "exporter.kubelet.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, le) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubelet }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100644 index 000000000..76215b399 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,568 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: +{{- if not (.Values.defaultRules.disabled.KubePodCrashLooping | default false) }} + - alert: KubePodCrashLooping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodcrashlooping + summary: Pod is crash looping. + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) >= 1 + for: {{ dig "KubePodCrashLooping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodCrashLooping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePodNotReady | default false) }} + - alert: KubePodNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepodnotready + summary: Pod has been in a non-ready state for more than 15 minutes. + expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown|Failed"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) group_left(owner_kind) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, cluster) ( + 1, max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: {{ dig "KubePodNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePodNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentGenerationMismatch | default false) }} + - alert: KubeDeploymentGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentgenerationmismatch + summary: Deployment generation mismatch due to possible roll-back + expr: |- + kube_deployment_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeDeploymentGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentReplicasMismatch | default false) }} + - alert: KubeDeploymentReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentreplicasmismatch + summary: Deployment has not matched the expected number of replicas. + expr: |- + ( + kube_deployment_spec_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_deployment_status_replicas_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeDeploymentReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDeploymentRolloutStuck | default false) }} + - alert: KubeDeploymentRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Rollout of deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} is not progressing for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedeploymentrolloutstuck + summary: Deployment rollout is not progressing. + expr: |- + kube_deployment_status_condition{condition="Progressing", status="false",job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != 0 + for: {{ dig "KubeDeploymentRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDeploymentRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetReplicasMismatch | default false) }} + - alert: KubeStatefulSetReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetreplicasmismatch + summary: StatefulSet has not matched the expected number of replicas. + expr: |- + ( + kube_statefulset_status_replicas_ready{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[10m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetGenerationMismatch | default false) }} + - alert: KubeStatefulSetGenerationMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetgenerationmismatch + summary: StatefulSet generation mismatch due to possible roll-back + expr: |- + kube_statefulset_status_observed_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeStatefulSetGenerationMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetGenerationMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeStatefulSetUpdateNotRolledOut | default false) }} + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubestatefulsetupdatenotrolledout + summary: StatefulSet update has not been rolled out. + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeStatefulSetUpdateNotRolledOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeStatefulSetUpdateNotRolledOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetRolloutStuck | default false) }} + - alert: KubeDaemonSetRolloutStuck + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetrolloutstuck + summary: DaemonSet rollout is stuck. + expr: |- + ( + ( + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + 0 + ) or ( + kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) or ( + kube_daemonset_status_number_available{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_daemonset_status_updated_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: {{ dig "KubeDaemonSetRolloutStuck" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetRolloutStuck" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeContainerWaiting | default false) }} + - alert: KubeContainerWaiting + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontainerwaiting + summary: Pod container waiting longer than 1 hour + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) > 0 + for: {{ dig "KubeContainerWaiting" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeContainerWaiting" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetNotScheduled | default false) }} + - alert: KubeDaemonSetNotScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetnotscheduled + summary: DaemonSet pods are not scheduled. + expr: |- + kube_daemonset_status_desired_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetNotScheduled" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetNotScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeDaemonSetMisScheduled | default false) }} + - alert: KubeDaemonSetMisScheduled + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubedaemonsetmisscheduled + summary: DaemonSet pods are misscheduled. + expr: kube_daemonset_status_number_misscheduled{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeDaemonSetMisScheduled" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeDaemonSetMisScheduled" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobNotCompleted | default false) }} + - alert: KubeJobNotCompleted + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobnotcompleted + summary: Job did not complete in time + expr: |- + time() - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}namespace, job_name, cluster) (kube_job_status_start_time{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + and + kube_job_status_active{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0) > 43200 + labels: + severity: {{ dig "KubeJobNotCompleted" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeJobFailed | default false) }} + - alert: KubeJobFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. Removing failed job after investigation should clear this alert. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubejobfailed + summary: Job failed to complete. + expr: kube_job_failed{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} > 0 + for: {{ dig "KubeJobFailed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeJobFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaReplicasMismatch | default false) }} + - alert: KubeHpaReplicasMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpareplicasmismatch + summary: HPA has not matched desired number of replicas. + expr: |- + (kube_horizontalpodautoscaler_status_desired_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + for: {{ dig "KubeHpaReplicasMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaReplicasMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeHpaMaxedOut | default false) }} + - alert: KubeHpaMaxedOut + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesApps | indent 8 }} +{{- end }} + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubehpamaxedout + summary: HPA is running at max replicas + expr: |- + kube_horizontalpodautoscaler_status_current_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="{{ $kubeStateMetricsJob }}", namespace=~"{{ $targetNamespace }}"} + for: {{ dig "KubeHpaMaxedOut" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeHpaMaxedOut" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesApps }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100644 index 000000000..911128525 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,282 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: +{{- if not (.Values.defaultRules.disabled.KubeCPUOvercommit | default false) }} + - alert: KubeCPUOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Pods by {{`{{`}} $value {{`}}`}} CPU shares and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="{{ $kubeStateMetricsJob }}",}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{job="{{ $kubeStateMetricsJob }}",resource="cpu"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeCPUOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryOvercommit | default false) }} + - alert: KubeMemoryOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + and + (sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) - max(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster)) > 0 + for: {{ dig "KubeMemoryOvercommit" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeCPUQuotaOvercommit | default false) }} + - alert: KubeCPUQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecpuquotaovercommit + summary: Cluster has overcommitted CPU resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(cpu|requests.cpu)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="cpu", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeCPUQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeCPUQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeMemoryQuotaOvercommit | default false) }} + - alert: KubeMemoryQuotaOvercommit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Cluster {{`{{`}} $labels.cluster {{`}}`}} has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubememoryquotaovercommit + summary: Cluster has overcommitted memory resource requests. + expr: |- + sum(min without(resource) (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard", resource=~"(memory|requests.memory)"})) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + / + sum(kube_node_status_allocatable{resource="memory", job="{{ $kubeStateMetricsJob }}"}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + > 1.5 + for: {{ dig "KubeMemoryQuotaOvercommit" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeMemoryQuotaOvercommit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaAlmostFull | default false) }} + - alert: KubeQuotaAlmostFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaalmostfull + summary: Namespace quota is going to be full. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 0.9 < 1 + for: {{ dig "KubeQuotaAlmostFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaAlmostFull" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaFullyUsed | default false) }} + - alert: KubeQuotaFullyUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotafullyused + summary: Namespace quota is fully used. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + == 1 + for: {{ dig "KubeQuotaFullyUsed" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaFullyUsed" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeQuotaExceeded | default false) }} + - alert: KubeQuotaExceeded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubequotaexceeded + summary: Namespace quota has exceeded the limits. + expr: |- + kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="{{ $kubeStateMetricsJob }}", type="hard"} > 0) + > 1 + for: {{ dig "KubeQuotaExceeded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeQuotaExceeded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.CPUThrottlingHigh | default false) }} + - alert: CPUThrottlingHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesResources | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/cputhrottlinghigh + summary: Processes experience elevated CPU throttling. + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, container, pod, namespace) + > ( 25 / 100 ) + for: {{ dig "CPUThrottlingHigh" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "CPUThrottlingHigh" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesResources }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100644 index 000000000..809e54488 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,216 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeFillingUp | default false) }} + - alert: KubePersistentVolumeFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeInodesFillingUp | default false) }} + - alert: KubePersistentVolumeInodesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: |- + ( + kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_inodes{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_inodes_used{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_inodes_free{job="{{ include "exporter.kubelet.jobName" . }}", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: {{ dig "KubePersistentVolumeInodesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeInodesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubePersistentVolumeErrors | default false) }} + - alert: KubePersistentVolumeErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesStorage | indent 8 }} +{{- end }} + description: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} {{`{{`}} with $labels.cluster -{{`}}`}} on Cluster {{`{{`}} . {{`}}`}} {{`{{`}}- end {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubepersistentvolumeerrors + summary: PersistentVolume is having issues with provisioning. + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="{{ $kubeStateMetricsJob }}"} > 0 + for: {{ dig "KubePersistentVolumeErrors" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubePersistentVolumeErrors" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesStorage }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100644 index 000000000..6dd61b5f5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,193 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientCertificateExpiration | default false) }} + - alert: KubeClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclientcertificateexpiration + summary: Client certificate is about to expire. + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job) histogram_quantile(0.01, sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + for: {{ dig "KubeClientCertificateExpiration" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIErrors | default false) }} + - alert: KubeAggregatedAPIErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4 + labels: + severity: {{ dig "KubeAggregatedAPIErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAggregatedAPIDown | default false) }} + - alert: KubeAggregatedAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: (1 - max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85 + for: {{ dig "KubeAggregatedAPIDown" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAggregatedAPIDown" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if .Values.kubeApiServer.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeAPIDown | default false) }} + - alert: KubeAPIDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapidown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="apiserver"} == 1) + for: {{ dig "KubeAPIDown" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPIDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeAPITerminatedRequests | default false) }} + - alert: KubeAPITerminatedRequests + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeapiterminatedrequests + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: {{ dig "KubeAPITerminatedRequests" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeAPITerminatedRequests" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100644 index 000000000..43b324596 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeControllerManager }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if .Values.kubeControllerManager.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeControllerManagerDown | default false) }} + - alert: KubeControllerManagerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeControllerManager | indent 8 }} +{{- end }} + description: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubecontrollermanagerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeControllerManagerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeControllerManager }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml new file mode 100644 index 000000000..2000acece --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kube-proxy.yaml @@ -0,0 +1,52 @@ +{{- /* +Generated from 'kubernetes-system-kube-proxy' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeProxy.enabled .Values.defaultRules.rules.kubeProxy }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kube-proxy" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kube-proxy + rules: +{{- if not (.Values.defaultRules.disabled.KubeProxyDown | default false) }} + - alert: KubeProxyDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeProxy | indent 8 }} +{{- end }} + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeProxy.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeProxyDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeProxy }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100644 index 000000000..d2cf87422 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,379 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: +{{- if not (.Values.defaultRules.disabled.KubeNodeNotReady | default false) }} + - alert: KubeNodeNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodenotready + summary: Node is not ready. + expr: kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",condition="Ready",status="true"} == 0 + for: {{ dig "KubeNodeNotReady" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeUnreachable | default false) }} + - alert: KubeNodeUnreachable + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodeunreachable + summary: Node is unreachable. + expr: (kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="{{ $kubeStateMetricsJob }}",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + for: {{ dig "KubeNodeUnreachable" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeUnreachable" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletTooManyPods | default false) }} + - alert: KubeletTooManyPods + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubelettoomanypods + summary: Kubelet is running at capacity. + expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + (kube_pod_status_phase{job="{{ $kubeStateMetricsJob }}",phase="Running"} == 1) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) group_left(node) topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}instance,pod,namespace,cluster) (1, kube_pod_info{job="{{ $kubeStateMetricsJob }}"}) + ) + / + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + kube_node_status_capacity{job="{{ $kubeStateMetricsJob }}",resource="pods"} != 1 + ) > 0.95 + for: {{ dig "KubeletTooManyPods" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletTooManyPods" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeNodeReadinessFlapping | default false) }} + - alert: KubeNodeReadinessFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubenodereadinessflapping + summary: Node readiness status is flapping. + expr: sum(changes(kube_node_status_condition{job="{{ $kubeStateMetricsJob }}",status="true",condition="Ready"}[15m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) > 2 + for: {{ dig "KubeNodeReadinessFlapping" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeNodeReadinessFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPlegDurationHigh | default false) }} + - alert: KubeletPlegDurationHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletplegdurationhigh + summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist. + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: {{ dig "KubeletPlegDurationHigh" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletPlegDurationHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletPodStartUpLatencyHigh | default false) }} + - alert: KubeletPodStartUpLatencyHigh + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletpodstartuplatencyhigh + summary: Kubelet Pod startup latency is too high. + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: {{ dig "KubeletPodStartUpLatencyHigh" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateExpiration | default false) }} + - alert: KubeletClientCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificateexpiration + summary: Kubelet client certificate is about to expire. + expr: kubelet_certificate_manager_client_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletClientCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 604800 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateExpiration | default false) }} + - alert: KubeletServerCertificateExpiration + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificateexpiration + summary: Kubelet server certificate is about to expire. + expr: kubelet_certificate_manager_server_ttl_seconds < 86400 + labels: + severity: {{ dig "KubeletServerCertificateExpiration" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletClientCertificateRenewalErrors | default false) }} + - alert: KubeletClientCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletclientcertificaterenewalerrors + summary: Kubelet has failed to renew its client certificate. + expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletClientCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletClientCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeletServerCertificateRenewalErrors | default false) }} + - alert: KubeletServerCertificateRenewalErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes). + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletservercertificaterenewalerrors + summary: Kubelet has failed to renew its server certificate. + expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0 + for: {{ dig "KubeletServerCertificateRenewalErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeletServerCertificateRenewalErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if (include "exporter.kubelet.enabled" .)}} +{{- if not (.Values.defaultRules.disabled.KubeletDown | default false) }} + - alert: KubeletDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeletdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubelet.jobName" . }}", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: {{ dig "KubeletDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100644 index 000000000..d32f15139 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if .Values.kubeScheduler.enabled }} +{{- if not (.Values.defaultRules.disabled.KubeSchedulerDown | default false) }} + - alert: KubeSchedulerDown + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubeSchedulerAlerting | indent 8 }} +{{- end }} + description: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeschedulerdown + summary: Target disappeared from Prometheus target discovery. + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: {{ dig "KubeSchedulerDown" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubeSchedulerAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100644 index 000000000..929a6f43b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,87 @@ +{{- /* +Generated from 'kubernetes-system' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: +{{- if not (.Values.defaultRules.disabled.KubeVersionMismatch | default false) }} + - alert: KubeVersionMismatch + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeversionmismatch + summary: Different semantic versions of Kubernetes components running. + expr: count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + for: {{ dig "KubeVersionMismatch" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeVersionMismatch" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.KubeClientErrors | default false) }} + - alert: KubeClientErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.kubernetesSystem | indent 8 }} +{{- end }} + description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/kubernetes/kubeclienterrors + summary: Kubernetes API server client is experiencing errors. + expr: |- + (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace) + / + sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, job, namespace)) + > 0.01 + for: {{ dig "KubeClientErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "KubeClientErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.kubernetesSystem }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100644 index 000000000..aeaa80231 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,188 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu, mode) ( + node_cpu_seconds_total{job="node-exporter",mode="idle"} + ) + record: instance:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg without (cpu) ( + sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m])) + ) + record: instance:node_cpu_utilisation:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - ( + ( + node_memory_MemAvailable_bytes{job="node-exporter"} + or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + + node_memory_Cached_bytes{job="node-exporter"} + + + node_memory_MemFree_bytes{job="node-exporter"} + + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterRecording }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100644 index 000000000..80bfd9bf3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,801 @@ +{{- /* +Generated from 'node-exporter' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.nodeExporterAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemSpaceFillingUp | default false) }} + - alert: NodeFilesystemSpaceFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemSpaceFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemSpaceFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfSpace | default false) }} + - alert: NodeFilesystemAlmostOutOfSpace + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfSpace" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfSpace" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemFilesFillingUp | default false) }} + - alert: NodeFilesystemFilesFillingUp + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemFilesFillingUp" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemFilesFillingUp" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFilesystemAlmostOutOfFiles | default false) }} + - alert: NodeFilesystemAlmostOutOfFiles + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0 + ) + for: {{ dig "NodeFilesystemAlmostOutOfFiles" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFilesystemAlmostOutOfFiles" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkReceiveErrs | default false) }} + - alert: NodeNetworkReceiveErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkReceiveErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkReceiveErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeNetworkTransmitErrs | default false) }} + - alert: NodeNetworkTransmitErrs + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 + for: {{ dig "NodeNetworkTransmitErrs" "for" "1h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkTransmitErrs" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeHighNumberConntrackEntriesUsed | default false) }} + - alert: NodeHighNumberConntrackEntriesUsed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: {{ dig "NodeHighNumberConntrackEntriesUsed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeTextFileCollectorScrapeError | default false) }} + - alert: NodeTextFileCollectorScrapeError + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: {{ dig "NodeTextFileCollectorScrapeError" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockSkewDetected | default false) }} + - alert: NodeClockSkewDetected + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds{job="node-exporter"} > 0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds{job="node-exporter"} < -0.05 + and + deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0 + ) + for: {{ dig "NodeClockSkewDetected" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockSkewDetected" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeClockNotSynchronising | default false) }} + - alert: NodeClockNotSynchronising + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodeclocknotsynchronising + summary: Clock not synchronising. + expr: |- + min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 + and + node_timex_maxerror_seconds{job="node-exporter"} >= 16 + for: {{ dig "NodeClockNotSynchronising" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeClockNotSynchronising" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDegraded | default false) }} + - alert: NodeRAIDDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddegraded + summary: RAID Array is degraded. + expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0 + for: {{ dig "NodeRAIDDegraded" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeRAIDDegraded" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeRAIDDiskFailure | default false) }} + - alert: NodeRAIDDiskFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/noderaiddiskfailure + summary: Failed device in RAID array. + expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0 + labels: + severity: {{ dig "NodeRAIDDiskFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeFileDescriptorLimit | default false) }} + - alert: NodeFileDescriptorLimit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodefiledescriptorlimit + summary: Kernel is predicted to exhaust file descriptors limit soon. + expr: |- + ( + node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90 + ) + for: {{ dig "NodeFileDescriptorLimit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeFileDescriptorLimit" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeCPUHighUsage | default false) }} + - alert: NodeCPUHighUsage + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodecpuhighusage + summary: High CPU usage. + expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: {{ dig "NodeCPUHighUsage" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeCPUHighUsage" "severity" "info" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemSaturation | default false) }} + - alert: NodeSystemSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This might indicate this instance resources saturation and can cause it becoming unresponsive. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemsaturation + summary: System saturated, load per core is very high. + expr: |- + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: {{ dig "NodeSystemSaturation" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryMajorPagesFaults | default false) }} + - alert: NodeMemoryMajorPagesFaults + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + Please check that there is enough memory available at this instance. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememorymajorpagesfaults + summary: Memory major page faults are occurring at very high rate. + expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: {{ dig "NodeMemoryMajorPagesFaults" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryMajorPagesFaults" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeMemoryHighUtilization | default false) }} + - alert: NodeMemoryHighUtilization + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodememoryhighutilization + summary: Host is running out of memory. + expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: {{ dig "NodeMemoryHighUtilization" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeMemoryHighUtilization" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeDiskIOSaturation | default false) }} + - alert: NodeDiskIOSaturation + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: 'Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 30 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + + This symptom might indicate disk saturation. + + ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodediskiosaturation + summary: Disk IO queue is high. + expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10 + for: {{ dig "NodeDiskIOSaturation" "for" "30m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeDiskIOSaturation" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeSystemdServiceFailed | default false) }} + - alert: NodeSystemdServiceFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodesystemdservicefailed + summary: Systemd service has entered failed state. + expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: {{ dig "NodeSystemdServiceFailed" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeSystemdServiceFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.NodeBondingDegraded | default false) }} + - alert: NodeBondingDegraded + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.nodeExporterAlerting | indent 8 }} +{{- end }} + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/node/nodebondingdegraded + summary: Bonding interface is degraded + expr: (node_bonding_slaves - node_bonding_active) != 0 + for: {{ dig "NodeBondingDegraded" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeBondingDegraded" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.nodeExporterAlerting }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-network.yaml new file mode 100644 index 000000000..16690ee31 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,55 @@ +{{- /* +Generated from 'node-network' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: +{{- if not (.Values.defaultRules.disabled.NodeNetworkInterfaceFlapping | default false) }} + - alert: NodeNetworkInterfaceFlapping + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.network }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.network | indent 8 }} +{{- end }} + description: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing its up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/general/nodenetworkinterfaceflapping + summary: Network interface is often changing its status + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: {{ dig "NodeNetworkInterfaceFlapping" "for" "2m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "NodeNetworkInterfaceFlapping" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.network }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100644 index 000000000..0bf9b8653 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'node.rules' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: |- + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node, namespace, pod) ( + label_replace(kube_pod_info{job="{{ $kubeStateMetricsJob }}",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + node_cpu_seconds_total{mode="idle",job="node-exporter"} + * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) group_left(node) + topk by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:) + ) + record: node:node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) + record: :node_memory_MemAvailable_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, node) ( + sum without (mode) ( + rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m]) + ) + ) + record: node:node_cpu_utilization:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + node:node_cpu_utilization:ratio_rate5m + ) + record: cluster:node_cpu:ratio_rate5m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.node }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100644 index 000000000..cb5c49587 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,253 @@ +{{- /* +Generated from 'prometheus-operator' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorListErrors | default false) }} + - alert: PrometheusOperatorListErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorlisterrors + summary: Errors while performing list operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[10m]))) > 0.4 + for: {{ dig "PrometheusOperatorListErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorListErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorWatchErrors | default false) }} + - alert: PrometheusOperatorWatchErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorwatcherrors + summary: Errors while performing watch operations in controller. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m])) / sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.4 + for: {{ dig "PrometheusOperatorWatchErrors" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorWatchErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorSyncFailed | default false) }} + - alert: PrometheusOperatorSyncFailed + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Controller {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} namespace fails to reconcile {{`{{`}} $value {{`}}`}} objects. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorsyncfailed + summary: Last controller reconciliation failed + expr: min_over_time(prometheus_operator_syncs{status="failed",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorSyncFailed" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorSyncFailed" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorReconcileErrors | default false) }} + - alert: PrometheusOperatorReconcileErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of reconciling operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorreconcileerrors + summary: Errors while reconciling objects. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorReconcileErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorReconcileErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorStatusUpdateErrors | default false) }} + - alert: PrometheusOperatorStatusUpdateErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of status update operations failed for {{`{{`}} $labels.controller {{`}}`}} controller in {{`{{`}} $labels.namespace {{`}}`}} namespace.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorstatusupdateerrors + summary: Errors while updating objects status. + expr: (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) / (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]))) > 0.1 + for: {{ dig "PrometheusOperatorStatusUpdateErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorStatusUpdateErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNodeLookupErrors | default false) }} + - alert: PrometheusOperatorNodeLookupErrors + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornodelookuperrors + summary: Errors while reconciling Prometheus. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: {{ dig "PrometheusOperatorNodeLookupErrors" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNodeLookupErrors" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorNotReady | default false) }} + - alert: PrometheusOperatorNotReady + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatornotready + summary: Prometheus operator not ready + expr: min by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) == 0) + for: {{ dig "PrometheusOperatorNotReady" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorNotReady" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOperatorRejectedResources | default false) }} + - alert: PrometheusOperatorRejectedResources + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheusOperator | indent 8 }} +{{- end }} + description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace rejected {{`{{`}} printf "%0.0f" $value {{`}}`}} {{`{{`}} $labels.controller {{`}}`}}/{{`{{`}} $labels.resource {{`}}`}} resources. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus-operator/prometheusoperatorrejectedresources + summary: Resources rejected by Prometheus operator + expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOperatorRejectedResources" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOperatorRejectedResources" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheusOperator }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100644 index 000000000..0cc617ff7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,735 @@ +{{- /* +Generated from 'prometheus' group from https://github.com/prometheus-operator/kube-prometheus.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: +{{- if not (.Values.defaultRules.disabled.PrometheusBadConfig | default false) }} + - alert: PrometheusBadConfig + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusbadconfig + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: {{ dig "PrometheusBadConfig" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusBadConfig" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusSDRefreshFailure | default false) }} + - alert: PrometheusSDRefreshFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to refresh SD with mechanism {{`{{`}}$labels.mechanism{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheussdrefreshfailure + summary: Failed Prometheus SD refresh. + expr: increase(prometheus_sd_refresh_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[10m]) > 0 + for: {{ dig "PrometheusSDRefreshFailure" "for" "20m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusSDRefreshFailure" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusKubernetesListWatchFailures | default false) }} + - alert: PrometheusKubernetesListWatchFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Kubernetes service discovery of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is experiencing {{`{{`}} printf "%.0f" $value {{`}}`}} failures with LIST/WATCH requests to the Kubernetes API in the last 5 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuskuberneteslistwatchfailures + summary: Requests in Kubernetes SD are failing. + expr: increase(prometheus_sd_kubernetes_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusKubernetesListWatchFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusKubernetesListWatchFailures" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotificationQueueRunningFull | default false) }} + - alert: PrometheusNotificationQueueRunningFull + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotificationqueuerunningfull + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusNotificationQueueRunningFull" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotificationQueueRunningFull" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToSomeAlertmanagers | default false) }} + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstosomealertmanagers + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToSomeAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotConnectedToAlertmanagers | default false) }} + - alert: PrometheusNotConnectedToAlertmanagers + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotconnectedtoalertmanagers + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: {{ dig "PrometheusNotConnectedToAlertmanagers" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotConnectedToAlertmanagers" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBReloadsFailing | default false) }} + - alert: PrometheusTSDBReloadsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbreloadsfailing + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBReloadsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBReloadsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTSDBCompactionsFailing | default false) }} + - alert: PrometheusTSDBCompactionsFailing + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustsdbcompactionsfailing + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: {{ dig "PrometheusTSDBCompactionsFailing" "for" "4h" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTSDBCompactionsFailing" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusNotIngestingSamples | default false) }} + - alert: PrometheusNotIngestingSamples + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusnotingestingsamples + summary: Prometheus is not ingesting samples. + expr: |- + ( + sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) + for: {{ dig "PrometheusNotIngestingSamples" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusNotIngestingSamples" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusDuplicateTimestamps | default false) }} + - alert: PrometheusDuplicateTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusduplicatetimestamps + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusDuplicateTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusDuplicateTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusOutOfOrderTimestamps | default false) }} + - alert: PrometheusOutOfOrderTimestamps + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusoutofordertimestamps + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusOutOfOrderTimestamps" "for" "10m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusOutOfOrderTimestamps" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteStorageFailures | default false) }} + - alert: PrometheusRemoteStorageFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotestoragefailures + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + / + ( + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + + + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + ) + ) + * 100 + > 1 + for: {{ dig "PrometheusRemoteStorageFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteStorageFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteBehind | default false) }} + - alert: PrometheusRemoteWriteBehind + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritebehind + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - ignoring(remote_name, url) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: {{ dig "PrometheusRemoteWriteBehind" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteBehind" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRemoteWriteDesiredShards | default false) }} + - alert: PrometheusRemoteWriteDesiredShards + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusremotewritedesiredshards + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: {{ dig "PrometheusRemoteWriteDesiredShards" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRemoteWriteDesiredShards" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusRuleFailures | default false) }} + - alert: PrometheusRuleFailures + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusrulefailures + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusRuleFailures" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusRuleFailures" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusMissingRuleEvaluations | default false) }} + - alert: PrometheusMissingRuleEvaluations + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusmissingruleevaluations + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusMissingRuleEvaluations" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusMissingRuleEvaluations" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetLimitHit | default false) }} + - alert: PrometheusTargetLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetlimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. + expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusTargetLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusLabelLimitHit | default false) }} + - alert: PrometheusLabelLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusLabelLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusLabelLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeBodySizeLimitHit | default false) }} + - alert: PrometheusScrapeBodySizeLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeBodySizeLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeBodySizeLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusScrapeSampleLimitHit | default false) }} + - alert: PrometheusScrapeSampleLimitHit + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: {{ dig "PrometheusScrapeSampleLimitHit" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusScrapeSampleLimitHit" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusTargetSyncFailure | default false) }} + - alert: PrometheusTargetSyncFailure + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: {{ dig "PrometheusTargetSyncFailure" "for" "5m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusTargetSyncFailure" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusHighQueryLoad | default false) }} + - alert: PrometheusHighQueryLoad + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} query API has less than 20% available capacity in its query engine for the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheushighqueryload + summary: Prometheus is reaching its maximum capacity serving concurrent requests. + expr: avg_over_time(prometheus_engine_queries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.8 + for: {{ dig "PrometheusHighQueryLoad" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusHighQueryLoad" "severity" "warning" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- if not (.Values.defaultRules.disabled.PrometheusErrorSendingAlertsToAnyAlertmanager | default false) }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: +{{- if .Values.defaultRules.additionalRuleAnnotations }} +{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }} +{{- end }} +{{- if .Values.defaultRules.additionalRuleGroupAnnotations.prometheus }} +{{ toYaml .Values.defaultRules.additionalRuleGroupAnnotations.prometheus | indent 8 }} +{{- end }} + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}/prometheus/prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "for" "15m" .Values.customRules }} + {{- with .Values.defaultRules.keepFiringFor }} + keep_firing_for: "{{ . }}" + {{- end }} + labels: + severity: {{ dig "PrometheusErrorSendingAlertsToAnyAlertmanager" "severity" "critical" .Values.customRules }} + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.prometheus }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.node.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.node.rules.yaml new file mode 100644 index 000000000..7c2555386 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.node.rules.yaml @@ -0,0 +1,301 @@ +{{- /* +Generated from 'windows.node.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.node.rules + rules: + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) ( + windows_system_system_up_time{job="windows-exporter"} + ) + record: node:windows_node:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + count by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) (sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, core) ( + windows_cpu_time_total{job="windows-exporter"} + )) + record: node:windows_node_num_cpu:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m])) + record: :windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + rate(windows_cpu_time_total{job="windows-exporter",mode="idle"}[1m]) + ) + record: node:windows_node_cpu_utilisation:avg1m + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + 1 - + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"}) + / + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: ':windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_memory_available_bytes{job="windows-exporter"} + windows_memory_cache_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemFreeCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: (windows_memory_cache_bytes{job="windows-exporter"} + windows_memory_modified_page_list_bytes{job="windows-exporter"} + windows_memory_standby_cache_core_bytes{job="windows-exporter"} + windows_memory_standby_cache_normal_priority_bytes{job="windows-exporter"} + windows_memory_standby_cache_reserve_bytes{job="windows-exporter"}) + record: node:windows_node_memory_totalCached_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (windows_os_visible_memory_bytes{job="windows-exporter"}) + record: :windows_node_memory_MemTotal_bytes:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (windows_memory_available_bytes{job="windows-exporter"}) + ) + record: node:windows_node_memory_bytes_available:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + windows_os_visible_memory_bytes{job="windows-exporter"} + ) + record: node:windows_node_memory_bytes_total:sum + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + (node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum) + / + scalar(sum(node:windows_node_memory_bytes_total:sum)) + record: node:windows_node_memory_utilisation:ratio + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: 1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum) + record: 'node:windows_node_memory_utilisation:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: irate(windows_memory_swap_page_operations_total{job="windows-exporter"}[5m]) + record: node:windows_node_memory_swap_io_pages:irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m]) + ) + record: :windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + avg by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_logical_disk_read_seconds_total{job="windows-exporter"}[1m]) + + irate(windows_logical_disk_write_seconds_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_disk_utilisation:avg_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster,instance,volume)( + (windows_logical_disk_size_bytes{job="windows-exporter"} + - windows_logical_disk_free_bytes{job="windows-exporter"}) + / windows_logical_disk_size_bytes{job="windows-exporter"} + ) + record: 'node:windows_node_filesystem_usage:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance, volume) (windows_logical_disk_free_bytes{job="windows-exporter"} / windows_logical_disk_size_bytes{job="windows-exporter"}) + record: 'node:windows_node_filesystem_avail:' + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + record: :windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_bytes_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_utilisation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m])) + + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster) (irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + record: :windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, instance) ( + (irate(windows_net_packets_received_discarded_total{job="windows-exporter"}[1m]) + + irate(windows_net_packets_outbound_discarded_total{job="windows-exporter"}[1m])) + ) + record: node:windows_node_net_saturation:sum_irate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.pod.rules.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.pod.rules.yaml new file mode 100644 index 000000000..86340b5c0 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/rules-1.14/windows.pod.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'windows.pod.rules' group from https://github.com/kubernetes-monitoring/kubernetes-mixin.git +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.windowsMonitoring.enabled .Values.defaultRules.rules.windows }} +{{- $kubeStateMetricsJob := include "kube-prometheus-stack-kube-state-metrics.name" . }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "windows.pod.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: windows.pod.rules + rules: + - expr: windows_container_available{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_pod_container_available + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_cpu_usage_seconds_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_total_runtime + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_commit_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_memory_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_memory_usage_private_working_set_bytes{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_private_working_set_usage + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_receive_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_received_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: windows_container_network_transmit_bytes_total{job="windows-exporter", container_id != ""} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job="{{ $kubeStateMetricsJob }}", container_id != ""}) by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container, container_id, pod, namespace, cluster) + record: windows_container_network_transmitted_bytes_total + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="memory",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="memory",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_memory_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + max by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + kube_pod_container_resource_requests{resource="cpu",job="{{ $kubeStateMetricsJob }}"} + ) * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_request + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: kube_pod_container_resource_limits{resource="cpu",job="{{ $kubeStateMetricsJob }}"} * on ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}container,pod,namespace,cluster) (windows_pod_container_available) + record: kube_pod_windows_container_resource_cpu_cores_limit + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + - expr: |- + sum by ({{ range $.Values.defaultRules.additionalAggregationLabels }}{{ . }},{{ end }}cluster, namespace, pod, container) ( + rate(windows_container_total_runtime{}[5m]) + ) + record: namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate + {{- if or .Values.defaultRules.additionalRuleLabels .Values.defaultRules.additionalRuleGroupLabels.windows }} + labels: + {{- with .Values.defaultRules.additionalRuleLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.defaultRules.additionalRuleGroupLabels.windows }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/secret.yaml new file mode 100644 index 000000000..e4a1e73c7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.objectStorageConfig}} +{{- if and .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret (not .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + object-storage-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/service.yaml new file mode 100644 index 000000000..bfabebe7b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/service.yaml @@ -0,0 +1,84 @@ +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.prometheus.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.prometheus.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} + - name: reloader-web + {{- if semverCompare "> 1.20.0-0" $kubeTargetVersion }} + appProtocol: http + {{- end }} + port: {{ .Values.prometheus.service.reloaderWebPort }} + targetPort: reloader-web + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} + selector: + {{- if .Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + {{- else }} + app.kubernetes.io/name: prometheus + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} +{{- if eq .Values.prometheus.service.sessionAffinity "ClientIP" }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.prometheus.service.sessionAffinityConfig.clientIP.timeoutSeconds }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecar.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecar.yaml new file mode 100644 index 000000000..87fae7b4f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecar.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosService.type }} + clusterIP: {{ .Values.prometheus.thanosService.clusterIP }} +{{- if .Values.prometheus.thanosService.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.prometheus.thanosService.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.prometheus.thanosService.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosService.httpPortName }} + port: {{ .Values.prometheus.thanosService.httpPort }} + targetPort: {{ .Values.prometheus.thanosService.targetHttpPort }} + {{- if eq .Values.prometheus.thanosService.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosService.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 000000000..453eed7f1 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.thanosServiceExternal.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + - name: {{ .Values.prometheus.thanosServiceExternal.httpPortName }} + port: {{ .Values.prometheus.thanosServiceExternal.httpPort }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetHttpPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.httpNodePort }} + {{- end }} + selector: + app.kubernetes.io/name: prometheus + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceaccount.yaml new file mode 100644 index 000000000..e97b989bb --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +automountServiceAccountToken: {{ .Values.prometheus.serviceAccount.automountServiceAccountToken }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ include "kube-prometheus-stack.imagePullSecrets" . | trim | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitor.yaml new file mode 100644 index 000000000..a36f3e33c --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,97 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" + metricRelabelings: + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + - port: reloader-web + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "/metrics" + {{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.prometheus.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.prometheus.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.prometheus.serviceMonitor.interval .interval }} + interval: {{ default $.Values.prometheus.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.prometheus.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.prometheus.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.prometheus.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.prometheus.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.prometheus.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.prometheus.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.prometheus.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitorThanosSidecar.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitorThanosSidecar.yaml new file mode 100644 index 000000000..0f70aabb5 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitorThanosSidecar.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.thanosService.enabled .Values.prometheus.thanosServiceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-sidecar + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.thanosService.httpPortName }} + {{- if .Values.prometheus.thanosServiceMonitor.interval }} + interval: {{ .Values.prometheus.thanosServiceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.scheme }} + scheme: {{ .Values.prometheus.thanosServiceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.thanosServiceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.thanosServiceMonitor.bearerTokenFile }} + {{- end }} + path: "/metrics" + metricRelabelings: + {{- if .Values.prometheus.thanosServiceMonitor.metricRelabelings}} + {{ tpl (toYaml .Values.prometheus.thanosServiceMonitor.metricRelabelings | indent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.prometheus.thanosServiceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.thanosServiceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitors.yaml new file mode 100644 index 000000000..a7a301bab --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + {{- include "servicemonitor.scrapeLimits" . | nindent 6 }} + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .metricRelabelings }} + metricRelabelings: +{{ toYaml .metricRelabelings | indent 8 }} + {{- end }} + {{- if .relabelings }} + relabelings: +{{ toYaml .relabelings | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 000000000..3a88b2df3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.ipDualStack.enabled }} + ipFamilies: {{ toYaml $serviceValues.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ $serviceValues.ipDualStack.ipFamilyPolicy }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + {{- if ne $serviceValues.type "ClusterIP" }} + externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + {{- if $.Values.prometheus.agentMode }} + app.kubernetes.io/name: prometheus-agent + statefulset.kubernetes.io/pod-name: prom-agent-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- else }} + app.kubernetes.io/name: prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.prometheus.crname" $ }}-{{ $i }} + {{- end }} + operator.prometheus.io/name: {{ template "kube-prometheus-stack.prometheus.crname" $ }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/clusterrole.yaml new file mode 100644 index 000000000..56ca9f5ea --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,135 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - alertmanagerconfigs + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + - probes + - probes/finalizers + - alertmanagerconfigs + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" +{{- if .Values.grafana.enabled }} + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" +{{- end }} + verbs: + - 'get' + - 'create' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- if .Values.grafana.enabled }} + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +{{- end }} + resources: + - endpoints + verbs: + - list +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/config-role.yaml new file mode 100644 index 000000000..f48ffc827 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboard-role.yaml new file mode 100644 index 000000000..d2f81976a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml new file mode 100644 index 000000000..7b51a0bf7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/addons/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + {{- if .Values.grafana.sidecar.dashboards.annotations }} + annotations: {{ toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} + {{- end }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml new file mode 100644 index 000000000..d73b25745 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/cluster-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-cluster + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/cluster/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml new file mode 100644 index 000000000..8865efa93 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-home + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/home/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml new file mode 100644 index 000000000..9b05cea2e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fleet-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fleet-dashboards + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/fleet/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentbit-dashboard.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentbit-dashboard.yaml new file mode 100644 index 000000000..b2d1bfcb7 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentbit-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.loggingMonitors.fluentbit.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fluentbit-dashboard + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/logging/fluentbit.json").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentd-dashboard.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentd-dashboard.yaml new file mode 100644 index 000000000..66c9cb845 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/fluentd-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.loggingMonitors.fluentd.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-fluentd-dashboard + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/logging/fluentd.json").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml new file mode 100644 index 000000000..2afae10ef --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/k8s-dashboards.yaml @@ -0,0 +1,31 @@ +{{- $files := (.Files.Glob "files/rancher/k8s/*").AsConfig }} +{{- $filesDict := (fromYaml $files) }} +{{- if not (include "exporter.kubeEtcd.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-etcd-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-etcd.json") -}} +{{- end }} +{{- if not (include "exporter.kubeControllerManager.enabled" .) }} +{{- $filesDict = (unset $filesDict "rancher-k8s-components-nodes.json") -}} +{{- $filesDict = (unset $filesDict "rancher-k8s-components.json") -}} +{{- else }} +{{- $_ := (set $filesDict "rancher-k8s-components-nodes.json" (get $filesDict "rancher-k8s-components-nodes.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- $_ := (set $filesDict "rancher-k8s-components.json" (get $filesDict "rancher-k8s-components.json" | replace "kube-controller-manager" (include "exporter.kubeControllerManager.jobName" .))) -}} +{{- end }} +{{ $files = (toYaml $filesDict) }} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-k8s + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ $files | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml new file mode 100644 index 000000000..172c36e9d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/nodes-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-nodes + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/nodes/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml new file mode 100644 index 000000000..19836ec4e --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/performance-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.rancherMonitoring.enabled $selector }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-performance-debugging + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/performance/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml new file mode 100644 index 000000000..940f18869 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/pods-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-pods + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/pods/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml new file mode 100644 index 000000000..d146dacdd --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/dashboards/rancher/workload-dashboards.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: rancher-default-dashboards-workloads + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/workloads/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml new file mode 100644 index 000000000..90d24c206 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/fleet/servicemonitor.yaml @@ -0,0 +1,53 @@ +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-fleet-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: fleet + selector: + matchLabels: + app: fleet-controller +{{- end }} +--- +{{- if .Values.rancherMonitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: monitoring-gitops-controller + namespace: cattle-fleet-system +spec: + endpoints: + - port: metrics + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: gitops + selector: + matchLabels: + app: gitjob +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml new file mode 100644 index 000000000..07d3c03de --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/network-policy.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rke2IngressNginx.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + annotations: + np.rke2.io/ingress: resolved + name: rke2-ingress-network-policy + namespace: {{ include "rke2-ingress-nginx.namespace" . }} +spec: + ingress: + - ports: + - port: {{ .Values.rke2IngressNginx.metricsPort }} + protocol: TCP + podSelector: + matchLabels: + app.kubernetes.io/name: rke2-ingress-nginx + policyTypes: + - Ingress + {{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100644 index 000000000..53a9ad689 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,27 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100644 index 000000000..b0f92e63b --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,49 @@ +{{- if and (not .Values.ingressNginx.enabled) (.Values.rkeIngressNginx.enabled) }} +{{- fail "Cannot set .Values.rkeIngressNginx.enabled=true when .Values.ingressNginx.enabled=false" }} +{{- end }} +{{- if and .Values.ingressNginx.enabled (not .Values.rkeIngressNginx.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + {{- if .Values.ingressNginx.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.ingressNginx.serviceMonitor.proxyUrl}} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + metricRelabelings: + {{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + {{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml new file mode 100644 index 000000000..1fba8f23f --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/exporters/rancher/servicemonitor.yaml @@ -0,0 +1,58 @@ +{{- $selector := (include "rancher.serviceMonitor.selector" .) -}} +{{- if and .Values.rancherMonitoring.enabled $selector }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + name: rancher + namespace: cattle-system +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + port: http + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + serverName: rancher + metricRelabelings: + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + jobLabel: rancher +{{- if .Values.rancherMonitoring.namespaceSelector }} + namespaceSelector: {{ .Values.rancherMonitoring.namespaceSelector | toYaml | nindent 4 }} +{{- end }} + selector: {{ include "rancher.serviceMonitor.selector" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +rules: +- apiGroups: + - management.cattle.io + resources: + - ranchermetrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-rancher-metrics +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/hardened.yaml new file mode 100644 index 000000000..63bac7fd4 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,91 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +{{- if .Values.hardened.k3s.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rancher-monitoring-coredns-allow-all + namespace: kube-system +spec: + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + k8s-app: kube-dns +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/configmap.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/configmap.yaml new file mode 100644 index 000000000..53cb89821 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "0" +data: +{{ (.Files.Glob "files/upgrade/scripts/*").AsConfig | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/job.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/job.yaml new file mode 100644 index 000000000..8f2771740 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/job.yaml @@ -0,0 +1,46 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "2" +spec: + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + spec: + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + securityContext: + runAsNonRoot: false + runAsUser: 0 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + - name: run-scripts + image: {{ template "system_default_registry" . }}{{ .Values.upgrade.image.repository }}:{{ .Values.upgrade.image.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: + - /bin/sh + - -c + - > + for s in $(find /etc/scripts -type f); do + echo "Running $s..."; + cat $s | bash + done; + volumeMounts: + - name: upgrade + mountPath: /etc/scripts + volumes: + - name: upgrade + configMap: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/rbac.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/rbac.yaml new file mode 100644 index 000000000..46bdd3a36 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/rancher-monitoring/upgrade/rbac.yaml @@ -0,0 +1,86 @@ +{{- if .Values.upgrade.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded + "helm.sh/hook-weight": "1" +rules: +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - 'list' + - 'delete' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +rules: +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.fullname" . }}-upgrade + annotations: + "helm.sh/hook": pre-upgrade, pre-rollback + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed + "helm.sh/hook-weight": "1" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/extrasecret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/extrasecret.yaml new file mode 100644 index 000000000..587fca2dc --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.thanosRuler.extraSecret.data -}} +{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.thanosRuler.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.extraSecret.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.thanosRuler.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ingress.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ingress.yaml new file mode 100644 index 000000000..e245ad448 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ingress.yaml @@ -0,0 +1,77 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }} +{{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }} +{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }} +{{- $servicePort := .Values.thanosRuler.service.port -}} +{{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }} +{{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.thanosRuler.ingress.annotations }} + annotations: + {{- tpl (toYaml .Values.thanosRuler.ingress.annotations) . | nindent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- if .Values.thanosRuler.ingress.labels }} +{{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if $apiIsStable }} + {{- if .Values.thanosRuler.ingress.ingressClassName }} + ingressClassName: {{ .Values.thanosRuler.ingress.ingressClassName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.thanosRuler.ingress.hosts }} + {{- range $host := .Values.thanosRuler.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + {{- if and $pathType $ingressSupportsPathType }} + pathType: {{ $pathType }} + {{- end }} + backend: + {{- if $apiIsStable }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- end -}} + {{- end -}} + {{- if .Values.thanosRuler.ingress.tls }} + tls: +{{ tpl (toYaml .Values.thanosRuler.ingress.tls | indent 4) . }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/podDisruptionBudget.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/podDisruptionBudget.yaml new file mode 100644 index 000000000..c28f91464 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.podDisruptionBudget.enabled }} +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.thanosRuler.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.thanosRuler.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.crname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ruler.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ruler.yaml new file mode 100644 index 000000000..94dc60be3 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/ruler.yaml @@ -0,0 +1,200 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ThanosRuler +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.crname" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.thanosRulerSpec.image }} + {{- $registry := include "monitoring_registry" . | default .Values.thanosRuler.thanosRulerSpec.image.registry -}} + {{- if and .Values.thanosRuler.thanosRulerSpec.image.tag .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.sha }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}@sha256:{{ .Values.thanosRuler.thanosRulerSpec.image.sha }}" + {{- else if .Values.thanosRuler.thanosRulerSpec.image.tag }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}:{{ .Values.thanosRuler.thanosRulerSpec.image.tag }}" + {{- else }} + image: "{{ $registry }}/{{ .Values.thanosRuler.thanosRulerSpec.image.repository }}" + {{- end }} + {{- if .Values.thanosRuler.thanosRulerSpec.image.sha }} + sha: {{ .Values.thanosRuler.thanosRulerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.thanosRuler.thanosRulerSpec.replicas }} + listenLocal: {{ .Values.thanosRuler.thanosRulerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} +{{- if .Values.thanosRuler.thanosRulerSpec.externalPrefix }} + externalPrefix: "{{ tpl .Values.thanosRuler.thanosRulerSpec.externalPrefix . }}" +{{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }} + externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.externalPrefixNilUsesHelmValues }} + externalPrefix: "http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }}" +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.additionalArgs }} + additionalArgs: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.additionalArgs) $ | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }} + nodeSelector: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.thanosRuler.thanosRulerSpec.paused }} + logFormat: {{ .Values.thanosRuler.thanosRulerSpec.logFormat | quote }} + logLevel: {{ .Values.thanosRuler.thanosRulerSpec.logLevel | quote }} + retention: {{ .Values.thanosRuler.thanosRulerSpec.retention | quote }} +{{- if .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} + evaluationInterval: {{ .Values.thanosRuler.thanosRulerSpec.evaluationInterval }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleNamespaceSelector | indent 4) . }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.ruleSelector }} + ruleSelector: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.ruleSelector | indent 4) .}} +{{- else if .Values.thanosRuler.thanosRulerSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }} + alertQueryUrl: "{{ .Values.thanosRuler.thanosRulerSpec.alertQueryUrl }}" +{{- end}} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl }} + alertmanagersUrl: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }} + alertmanagersConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }} + alertmanagersConfig: + key: alertmanager-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryEndpoints }} + queryEndpoints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }} + queryConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }} + queryConfig: + key: query-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.resources }} + resources: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.routePrefix }} + routePrefix: "{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}" +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.securityContext }} + securityContext: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.storage }} + storage: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }} + objectStorageConfig: + key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}" + name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}" +{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }} + objectStorageConfig: + key: object-storage-configs.yaml + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.labels }} + labels: +{{ tpl (toYaml .Values.thanosRuler.thanosRulerSpec.labels) $ | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.thanosRuler.thanosRulerSpec.podAntiAffinity .Values.thanosRuler.thanosRulerSpec.affinity }} + affinity: +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.affinity }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.crname" . }}]} +{{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.thanosRuler.thanosRulerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]} + - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.crname" . }}]} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.thanosRuler.thanosRulerSpec.tolerations }} +{{ toYaml .Values.thanosRuler.thanosRulerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.containers }} + containers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.initContainers }} + initContainers: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.priorityClassName }} + priorityClassName: {{.Values.thanosRuler.thanosRulerSpec.priorityClassName }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumes }} + volumes: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.volumeMounts | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.thanosRulerSpec.alertDropLabels }} + alertDropLabels: +{{ toYaml .Values.thanosRuler.thanosRulerSpec.alertDropLabels | indent 4 }} +{{- end }} + portName: {{ .Values.thanosRuler.thanosRulerSpec.portName }} +{{- with .Values.thanosRuler.thanosRulerSpec.additionalConfig }} + {{- tpl (toYaml .) $ | nindent 2 }} +{{- end }} +{{- with .Values.thanosRuler.thanosRulerSpec.additionalConfigString }} + {{- tpl . $ | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/secret.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/secret.yaml new file mode 100644 index 000000000..acab7fd9a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/secret.yaml @@ -0,0 +1,26 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ include "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }} + {{- if and .secret (not .existingSecret) }} + alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }} + {{- if and .secret (not .existingSecret) }} + object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} + {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }} + {{- if and .secret (not .existingSecret) }} + query-configs.yaml: {{ toYaml .secret | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/service.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/service.yaml new file mode 100644 index 000000000..e2cca2918 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/service.yaml @@ -0,0 +1,57 @@ +{{- if .Values.thanosRuler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.service.labels }} +{{ toYaml .Values.thanosRuler.service.labels | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.thanosRuler.service.clusterIP }} + clusterIP: {{ .Values.thanosRuler.service.clusterIP }} +{{- end }} +{{- if .Values.thanosRuler.service.ipDualStack.enabled }} + ipFamilies: {{ toYaml .Values.thanosRuler.service.ipDualStack.ipFamilies | nindent 4 }} + ipFamilyPolicy: {{ .Values.thanosRuler.service.ipDualStack.ipFamilyPolicy }} +{{- end }} +{{- if .Values.thanosRuler.service.externalIPs }} + externalIPs: +{{ toYaml .Values.thanosRuler.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.thanosRuler.service.loadBalancerIP }} +{{- end }} +{{- if .Values.thanosRuler.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.thanosRuler.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} +{{- if ne .Values.thanosRuler.service.type "ClusterIP" }} + externalTrafficPolicy: {{ .Values.thanosRuler.service.externalTrafficPolicy }} +{{- end }} + ports: + - name: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if eq .Values.thanosRuler.service.type "NodePort" }} + nodePort: {{ .Values.thanosRuler.service.nodePort }} + {{- end }} + port: {{ .Values.thanosRuler.service.port }} + targetPort: {{ .Values.thanosRuler.service.targetPort }} + protocol: TCP +{{- if .Values.thanosRuler.service.additionalPorts }} +{{ toYaml .Values.thanosRuler.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app.kubernetes.io/name: thanos-ruler + thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.crname" . }} + type: "{{ .Values.thanosRuler.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/serviceaccount.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/serviceaccount.yaml new file mode 100644 index 000000000..b58f1cd4d --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/serviceaccount.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + app.kubernetes.io/component: thanos-ruler +{{- include "kube-prometheus-stack.labels" . | indent 4 -}} +{{- if .Values.thanosRuler.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/servicemonitor.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/servicemonitor.yaml new file mode 100644 index 000000000..d26ddce93 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/thanos-ruler/servicemonitor.yaml @@ -0,0 +1,82 @@ +{{- if and .Values.thanosRuler.enabled .Values.thanosRuler.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: + {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.thanosRuler.name" . }} + release: {{ $.Release.Name | quote }} + self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.thanosRuler.thanosRulerSpec.portName }} + {{- if .Values.thanosRuler.serviceMonitor.interval }} + interval: {{ .Values.thanosRuler.serviceMonitor.interval }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.thanosRuler.serviceMonitor.proxyUrl}} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.scheme }} + scheme: {{ .Values.thanosRuler.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.thanosRuler.serviceMonitor.bearerTokenFile }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.tlsConfig }} + tlsConfig: {{- toYaml .Values.thanosRuler.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + path: "{{ trimSuffix "/" .Values.thanosRuler.thanosRulerSpec.routePrefix }}/metrics" + {{- if .Values.thanosRuler.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- tpl (toYaml .Values.thanosRuler.serviceMonitor.metricRelabelings | nindent 6) . }} + {{- end }} + {{ if .Values.global.cattle.clusterId }} + - sourceLabels: [__address__] + targetLabel: cluster_id + replacement: {{ .Values.global.cattle.clusterId }} + {{- end }} + {{ if .Values.global.cattle.clusterName}} + - sourceLabels: [__address__] + targetLabel: cluster_name + replacement: {{ .Values.global.cattle.clusterName }} + {{- end }} + {{- if .Values.thanosRuler.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.thanosRuler.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- range .Values.thanosRuler.serviceMonitor.additionalEndpoints }} + - port: {{ .port }} + {{- if or $.Values.thanosRuler.serviceMonitor.interval .interval }} + interval: {{ default $.Values.thanosRuler.serviceMonitor.interval .interval }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + proxyUrl: {{ default $.Values.thanosRuler.serviceMonitor.proxyUrl .proxyUrl }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + scheme: {{ default $.Values.thanosRuler.serviceMonitor.scheme .scheme }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + bearerTokenFile: {{ default $.Values.thanosRuler.serviceMonitor.bearerTokenFile .bearerTokenFile }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig }} + tlsConfig: {{- default $.Values.thanosRuler.serviceMonitor.tlsConfig .tlsConfig | toYaml | nindent 6 }} + {{- end }} + path: {{ .path }} + {{- if or $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings }} + metricRelabelings: {{- tpl (default $.Values.thanosRuler.serviceMonitor.metricRelabelings .metricRelabelings | toYaml | nindent 6) . }} + {{- end }} + {{- if or $.Values.thanosRuler.serviceMonitor.relabelings .relabelings }} + relabelings: {{- default $.Values.thanosRuler.serviceMonitor.relabelings .relabelings | toYaml | nindent 6 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-install-crd.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-install-crd.yaml new file mode 100644 index 000000000..6fcb8b3a6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-install-crd.yaml @@ -0,0 +1,23 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/AlertmanagerConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Probe" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/PrometheusAgent" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1alpha1/ScrapeConfig" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-psp-install.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-psp-install.yaml new file mode 100644 index 000000000..b115feb1a --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/templates/validate-psp-install.yaml @@ -0,0 +1,2 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- end }} diff --git a/charts/rancher-monitoring/106.0.0+up61.3.2/values.yaml b/charts/rancher-monitoring/106.0.0+up61.3.2/values.yaml new file mode 100644 index 000000000..e978703e6 --- /dev/null +++ b/charts/rancher-monitoring/106.0.0+up61.3.2/values.yaml @@ -0,0 +1,5567 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.23" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.23 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + seLinuxOptions: + # Gives rkeEtcd permissions to read files in /etc/kubernetes/* + # Type is defined in https://github.com/rancher/rancher-selinux + type: rke_kubereader_t + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeIngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + clients: + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + nodeSelector: + node-role.kubernetes.io/worker: "true" + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10250 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +hardened: + k3s: + networkPolicy: + enabled: true + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10257 # default to secure port as of k8s >= 1.22 + component: kube-controller-manager + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10252 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Scheduler: + enabled: false + metricsPort: 10259 # default to secure port as of k8s >= 1.22 + component: kube-scheduler + clients: + https: + enabled: true + insecureSkipVerify: true + useServiceAccountCredentials: true + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + kubeVersionOverrides: + - constraint: "< 1.22" + values: + metricsPort: 10251 # default to insecure port in k8s < 1.22 + clients: + https: + enabled: false + insecureSkipVerify: false + useServiceAccountCredentials: false + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2IngressNginx: + enabled: false + metricsPort: 10254 + component: ingress-nginx + networkPolicy: + enabled: false + # in the RKE2 cluster, the ingress-nginx-controller is deployed + # as a non-hostNetwork workload starting at the following versions + # - >= v1.22.12+rke2r1 < 1.23.0-0 + # - >= v1.23.9+rke2r1 < 1.24.0-0 + # - >= v1.24.3+rke2r1 < 1.25.0-0 + # - >= v1.25.0+rke2r1 + # As a result we do not need clients and proxies as we can directly create + # a service that targets the workload with the given app name + namespaceOverride: kube-system + clients: + enabled: false + proxy: + enabled: false + service: + selector: + app.kubernetes.io/name: rke2-ingress-nginx + kubeVersionOverrides: + - constraint: "< 1.21.0-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a DaemonSet with 1 pod when RKE2 version is < 1.21.0-0 + deployment: + enabled: false + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.21.0-0 < 1.22.12-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.21.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.23.0-0 < v1.23.9-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + - constraint: ">= 1.24.0-0 < v1.24.3-0" + values: + namespaceOverride: "" + clients: + enabled: true + port: 10015 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: "In" + values: + - "controller" + topologyKey: "kubernetes.io/hostname" + namespaces: + - "kube-system" + # in the RKE2 cluster, the ingress-nginx-controller is deployed as + # a hostNetwork Deployment with 1 pod when RKE2 version is >= 1.20.0-0 + deployment: + enabled: true + replicas: 1 + proxy: + enabled: true + service: + selector: false + + + +## Additional PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.9/packages/rancher-monitoring/rancher-pushprox +## + +# hardenedKubelet can only be deployed if kubelet.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default kubelet option with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedKubelet: + enabled: false + metricsPort: 10250 + component: kubelet + clients: + port: 10015 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + rbac: + additionalRules: + - nonResourceURLs: ["/metrics/cadvisor"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes/metrics"] + verbs: ["get"] + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + serviceMonitor: + endpoints: + - port: metrics + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/cadvisor + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + - port: metrics + path: /metrics/probes + honorLabels: true + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + +# hardenedNodeExporter can only be deployed if nodeExporter.enabled=true +# If enabled, it replaces the ServiceMonitor deployed by the default nodeExporter with a +# PushProx-based exporter that does not require a host port to be open to scrape metrics. +hardenedNodeExporter: + enabled: false + metricsPort: 9796 + component: node-exporter + clients: + port: 10016 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Upgrades +upgrade: + ## Run upgrade scripts before an upgrade or rollback via a Job hook + enabled: true + ## Image to use to run the scripts + image: + repository: rancher/shell + tag: v0.2.1 + +## Rancher Monitoring +## + +rancherMonitoring: + enabled: true + + ## A namespaceSelector to identify the namespace to find the Rancher deployment + ## + namespaceSelector: + matchNames: + - cattle-system + + ## A selector to identify the Rancher deployment + ## If not set, the chart will try to search for the Rancher deployment in the cattle-system namespace and infer the selector values from it + ## If the Rancher deployment does not exist, no resources will be deployed. + ## + selector: {} + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "30s" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.26.6 +## +kubeTargetVersionOverride: "" + +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Install Prometheus Operator CRDs +## +crds: + enabled: true + +## custom Rules to override "for" and "severity" in defaultRules +## +customRules: {} + # AlertmanagerFailedReload: + # for: 3m + # AlertmanagerMembersInconsistent: + # for: 5m + # severity: "warning" + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + configReloaders: true + general: true + k8sContainerCpuUsageSecondsTotal: true + k8sContainerMemoryCache: true + k8sContainerMemoryRss: true + k8sContainerMemorySwap: true + k8sContainerResource: true + k8sContainerMemoryWorkingSetBytes: true + k8sPodOwner: true + kubeApiserverAvailability: true + kubeApiserverBurnrate: true + kubeApiserverHistogram: true + kubeApiserverSlos: true + kubeControllerManager: true + kubelet: true + kubeProxy: true + kubePrometheusGeneral: true + kubePrometheusNodeRecording: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeSchedulerAlerting: true + kubeSchedulerRecording: true + kubeStateMetrics: true + network: true + node: true + nodeExporterAlerting: true + nodeExporterRecording: true + prometheus: true + prometheusOperator: true + windows: true + + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Set keep_firing_for for all alerts + keepFiringFor: "" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + + ## Additional labels for PrometheusRule alerts + additionalRuleLabels: {} + + ## Additional annotations for PrometheusRule alerts + additionalRuleAnnotations: {} + + ## Additional labels for specific PrometheusRule alert groups + additionalRuleGroupLabels: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + ## Additional annotations for specific PrometheusRule alerts groups + additionalRuleGroupAnnotations: + alertmanager: {} + etcd: {} + configReloaders: {} + general: {} + k8sContainerCpuUsageSecondsTotal: {} + k8sContainerMemoryCache: {} + k8sContainerMemoryRss: {} + k8sContainerMemorySwap: {} + k8sContainerResource: {} + k8sPodOwner: {} + kubeApiserverAvailability: {} + kubeApiserverBurnrate: {} + kubeApiserverHistogram: {} + kubeApiserverSlos: {} + kubeControllerManager: {} + kubelet: {} + kubeProxy: {} + kubePrometheusGeneral: {} + kubePrometheusNodeRecording: {} + kubernetesApps: {} + kubernetesResources: {} + kubernetesStorage: {} + kubernetesSystem: {} + kubeSchedulerAlerting: {} + kubeSchedulerRecording: {} + kubeStateMetrics: {} + network: {} + node: {} + nodeExporterAlerting: {} + nodeExporterRecording: {} + prometheus: {} + prometheusOperator: {} + + additionalAggregationLabels: [] + + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + +## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. +## +# additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRulesMap: {} +# rule-name: +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + + systemDefaultRegistry: "" + ## Windows Monitoring + ## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter + ## + ## Deploys a DaemonSet of Prometheus exporters based on https://github.com/prometheus-community/windows_exporter. + ## Every Windows host must have a wins version of 0.1.0+ to use this chart (default as of Rancher 2.5.8). + ## To upgrade wins versions on Windows hosts, see https://github.com/rancher/wins/tree/master/charts/rancher-wins-upgrader. + ## + windows: + enabled: false + seLinux: + enabled: false + kubectl: + repository: rancher/kuberlr-kubectl + tag: v4.0.0-rc.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + ## Create ClusterRoles that extend the existing view, edit and admin ClusterRoles to interact with prometheus-operator CRDs + ## Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles + createAggregateClusterRoles: false + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Global image registry to use if it needs to be overriden for some specific use cases (e.g local registries, custom images, ...) + ## + imageRegistry: "docker.io" + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + # or + # - "image-pull-secret" + +windowsMonitoring: + ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter') + enabled: false + +loggingMonitors: + ## Deploys logging-specific dashboards, make sure to also set metrics.serviceMonitor to true in the logging chart for both fluentd and fluentbit + fluentd: + enabled: false + fluentbit: + enabled: false + +## Configuration for prometheus-windows-exporter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter +## +prometheus-windows-exporter: + ## Enable ServiceMonitor and set Kubernetes label to use as a job label + ## + prometheus: + monitor: + enabled: true + jobLabel: jobLabel + + releaseLabel: true + + ## Set job label to 'windows-exporter' as required by the default Prometheus rules and Grafana dashboards + ## + podLabels: + jobLabel: windows-exporter + + ## Enable memory and container metrics as required by the default Prometheus rules and Grafana dashboards + ## + config: |- + collectors: + enabled: '[defaults],memory,container' + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Annotations for Alertmanager + ## + annotations: {} + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## @param alertmanager.enableFeatures Enable access to Alertmanager disabled features. + ## + enableFeatures: [] + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: + - 'severity = critical' + target_matchers: + - 'severity =~ warning|info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'severity = warning' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - 'alertname' + - source_matchers: + - 'alertname = InfoInhibitor' + target_matchers: + - 'severity = info' + equal: + - 'namespace' + - target_matchers: + - 'alertname = InfoInhibitor' + route: + group_by: ['namespace'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - receiver: 'null' + matchers: + - alertname = "Watchdog" + receivers: + - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' + + ## Alertmanager configuration directives (as string type, preferred over the config hash map) + ## stringConfig will be used only, if tplConfig is true + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + stringConfig: "" + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#tmpl_string + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + + labels: {} + + ## Override ingress to a different defined port on the service + # servicePort: 8081 + ## Override ingress to a different service then the default, this is useful if you need to + ## point to a specific instance of the alertmanager (eg kube-prometheus-stack-alertmanager-0) + # serviceName: kube-prometheus-stack-alertmanager-0 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + annotations: {} + + # by default the alertmanager secret is not overwritten if it already exists + recreateIfExists: false + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + + ## Additional ports to open for Alertmanager service + ## + additionalPorts: [] + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a ServiceMonitor for AlertManager + ## + serviceMonitor: + ## If true, a ServiceMonitor will be created for the AlertManager service. + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## enableHttp2: Whether to enable HTTP2. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint + enableHttp2: true + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prometheus-alertmanager + tag: v0.27.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerwebspec + web: {} + + ## AlertmanagerConfigs to be selected to merge and configure Alertmanager with. + ## + alertmanagerConfigSelector: {} + ## Example which selects all alertmanagerConfig resources + ## with label "alertconfig" with values any of "example-config" or "example-config-2" + # alertmanagerConfigSelector: + # matchExpressions: + # - key: alertconfig + # operator: In + # values: + # - example-config + # - example-config-2 + # + ## Example which selects all alertmanagerConfig resources with label "role" set to "example-config" + # alertmanagerConfigSelector: + # matchLabels: + # role: example-config + + ## Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace. + ## + alertmanagerConfigNamespaceSelector: {} + ## Example which selects all namespaces + ## with label "alertmanagerconfig" with values any of "example-namespace" or "example-namespace-2" + # alertmanagerConfigNamespaceSelector: + # matchExpressions: + # - key: alertmanagerconfig + # operator: In + # values: + # - example-namespace + # - example-namespace-2 + + ## Example which selects all namespaces with label "alertmanagerconfig" set to "enabled" + # alertmanagerConfigNamespaceSelector: + # matchLabels: + # alertmanagerconfig: enabled + + ## AlermanagerConfig to be used as top level configuration + ## + alertmanagerConfiguration: {} + ## Example with select a global alertmanagerconfig + # alertmanagerConfiguration: + # name: global-alertmanager-Configuration + + ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg: + ## + alertmanagerConfigMatcherStrategy: {} + ## Example with use OnNamespace strategy + # alertmanagerConfigMatcherStrategy: + # type: OnNamespace + + ## Define Log Format + # Use logfmt (default) or json logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## scheme: HTTP scheme to use. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when connect to the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + # containers: + # - name: oauth-proxy + # image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1 + # args: + # - --upstream=http://127.0.0.1:9093 + # - --http-address=0.0.0.0:8081 + # - --metrics-address=0.0.0.0:8082 + # - ... + # ports: + # - containerPort: 8081 + # name: oauth-proxy + # protocol: TCP + # - containerPort: 8082 + # name: oauth-metrics + # protocol: TCP + # resources: {} + + # Additional volumes on the output StatefulSet definition. + volumes: [] + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "http-web" + + ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 + ## + clusterAdvertiseAddress: false + + ## clusterGossipInterval determines interval between gossip attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterGossipInterval: "" + + ## clusterPeerTimeout determines timeout for cluster peering. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPeerTimeout: "" + + ## clusterPushpullInterval determines interval between pushpull attempts. + ## Needs to be specified as GoDuration, a time duration that can be parsed by Go’s time.ParseDuration() (e.g. 45ms, 30s, 1m, 1h20m15s) + clusterPushpullInterval: "" + + ## ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. + ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. + forceEnableClusterMode: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use alertmanager.alertmanagerSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + security: + # Required to embed dashboards in Rancher Cluster Overview Dashboard on Cluster Explorer + allow_embedding: true + + deploymentStrategy: + type: Recreate + + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + + ## Deploy default dashboards + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + ## Timezone for the default dashboards + ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg + ## + defaultDashboardsTimezone: utc + + ## Editable flag for the default dashboards + ## + defaultDashboardsEditable: true + + adminPassword: prom-operator + + rbac: + ## If true, Grafana PSPs will be created + ## + pspEnabled: false + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## IngressClassName for Grafana Ingress. + ## Should be provided if Ingress is enable. + ## + # ingressClassName: nginx + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + # # To make Grafana persistent (Using Statefulset) + # # + # persistence: + # enabled: true + # type: sts + # storageClassName: "storageClassName" + # accessModes: + # - ReadWriteOnce + # size: 20Gi + # finalizers: + # - kubernetes.io/pvc-protection + + serviceAccount: + create: true + autoMount: true + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + labelValue: "1" + + # Support for new table panels, when enabled grafana auto migrates the old table panels to newer table panels + enableNewTablePanelSyntax: false + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + + name: Prometheus + uid: prometheus + + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + + ## Prometheus request timeout in seconds + # timeout: 30 + + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Set method for HTTP to send query to datasource + httpMethod: POST + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/0fee93e12dc7c2ea1218f19ae25ec6b893460590/pkg/prometheus/statefulset.go#L255-L286 + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + ## Field with internal link pointing to existing data source in Grafana. + ## Can be provisioned via additionalDataSources + exemplarTraceIdDestinations: {} + # datasourceUid: Jaeger + # traceIdLabelName: trace_id + alertmanager: + enabled: true + name: Alertmanager + uid: alertmanager + handleGrafanaManagedAlerts: false + implementation: prometheus + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + ipFamilies: [] + ipFamilyPolicy: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + # If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + # + enabled: true + + # Path to use for scraping metrics. Might be different if server.root_url is set + # in grafana.ini + path: "/metrics" + + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + + # labels for the ServiceMonitor + labels: {} + + # Scrape interval. If not set, the Prometheus default scrape interval is used. + # + interval: "" + scheme: http + tlsConfig: {} + scrapeTimeout: 30s + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + + testFramework: + enabled: false + +## Flag to disable all the kubernetes component scrapers +## +kubernetesServiceMonitors: + enabled: true + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: + # Drop excessively noisy apiserver buckets. + - action: drop + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + sourceLabels: + - __name__ + - le + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Attach metadata to discovered targets. Requires Prometheus v2.45 for endpoints created by the operator. + ## + attachMetadata: + node: false + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## If true, Prometheus use (respect) labels provided by exporter. + ## + honorLabels: true + + ## If true, Prometheus ingests metrics with timestamp provided by exporter. If false, Prometheus ingests metrics with timestamp of scrape. + ## + honorTimestamps: true + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Skip TLS certificate validation when scraping. + ## This is enabled by default because kubelet serving certificate deployed by kubeadm is by default self-signed + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/#kubelet-serving-certs + ## + insecureSkipVerify: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## This is disabled by default because container metrics are already exposed by cAdvisor + ## + resource: false + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + cAdvisorMetricRelabelings: + # Drop less useful container CPU metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)' + # Drop less useful container / always zero filesystem metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)' + # Drop less useful / always zero container memory metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_memory_(mapped_file|swap)' + # Drop less useful container process metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_(file_descriptors|tasks_state|threads_max)' + # Drop container spec metrics that overlap with kube-state-metrics. + - sourceLabels: [__name__] + action: drop + regex: 'container_spec.*' + # Drop cgroup metrics with no pod. + - sourceLabels: [id, pod] + action: drop + regex: '.+;' + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + cAdvisorRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + probesRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + resourceRelabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + ## metrics_path is required to match upstream rules and charts + relabelings: + - action: replace + sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # component: kube-controller-manager + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-controller-manager + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + enabled: true + port: 9153 + targetPort: 9153 + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # k8s-app: kube-dns + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-dns + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + enabled: true + port: 2381 + targetPort: 2381 + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: etcd + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + enabled: true + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # component: kube-scheduler + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. + ## + https: null + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # component: kube-scheduler + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + enabled: true + port: 10249 + targetPort: 10249 + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + enabled: true + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## port: Name of the port the metrics will be scraped from + ## + port: http-metrics + + jobLabel: jobLabel + selector: {} + # matchLabels: + # k8s-app: kube-proxy + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## Additional labels + ## + additionalLabels: {} + # foo: bar + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + operatingSystems: + linux: + enabled: true + darwin: + enabled: true + + ## ForceDeployDashboard Create dashboard configmap even if nodeExporter deployment has been disabled + ## + forceDeployDashboards: false + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + releaseLabel: true + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + rbac: + ## If true, create PSPs for node-exporter + ## + pspEnabled: false + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-operator' by default + fullnameOverride: "" + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Strategy of the deployment + ## + strategy: {} + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + ## Valid values: Fail, Ignore, IgnoreOnInstallOnly + ## IgnoreOnInstallOnly - If Release.IsInstall returns "true", set "Ignore" otherwise "Fail" + failurePolicy: "" + ## The default timeoutSeconds is 10 and the maximum value is 30. + timeoutSeconds: 10 + enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + + namespaceSelector: {} + objectSelector: {} + + + deployment: + enabled: false + + ## Number of replicas + ## + replicas: 1 + + ## Strategy of the deployment + ## + strategy: {} + + # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ + podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1 + + ## Number of old replicasets to retain ## + ## The default value is 10, 0 will garbage-collect old replicasets ## + revisionHistoryLimit: 10 + + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## + tls: + enabled: true + # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants + tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 + + ## Service account for Prometheus Operator Webhook to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + automountServiceAccountToken: false + create: true + name: "" + + ## Configuration for Prometheus operator Webhook service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 31080 + + nodePortTls: 31443 + + ## Additional ports to open for Prometheus operator Webhook service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator webhook deployment + # ## + labels: {} + + ## Annotations to add to the operator webhook deployment + ## + annotations: {} + + ## Labels to add to the operator webhook pod + ## + podLabels: {} + + ## Annotations to add to the operator webhook pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## Prometheus-operator webhook image + ## + image: + registry: quay.io + repository: rancher/mirrored-prometheus-operator-admission-webhook + # if not set appVersion field from Chart.yaml is used + tag: v0.75.1 + sha: "" + pullPolicy: IfNotPresent + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + + ## Liveness probe + ## + livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Readiness probe + ## + readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + patch: + enabled: true + image: + repository: rancher/mirrored-ingress-nginx-kube-webhook-certgen + tag: v1.4.3 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + ttlSecondsAfterFinished: 60 + annotations: {} + # argocd.argoproj.io/hook: PreSync + # argocd.argoproj.io/hook-delete-policy: HookSucceeded + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + seccompProfile: + type: RuntimeDefault + ## Service account for Prometheus Operator Webhook Job Patch to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + automountServiceAccountToken: true + + # Security context for create job container + createSecretJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Security context for patch job container + patchWebhookJob: + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Filter namespaces to look for prometheus-operator custom resources + ## + alertmanagerInstanceNamespaces: [] + alertmanagerConfigNamespaces: [] + prometheusInstanceNamespaces: [] + thanosRulerInstanceNamespaces: [] + + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + + networkPolicy: + ## Enable creation of NetworkPolicy resources. + ## + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # egress: + + ## match labels used in selector + # matchLabels: {} + + ## Service account for Prometheus Operator to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + automountServiceAccountToken: true + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus operator service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## NodePort, ClusterIP, LoadBalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # ## Labels to add to the operator deployment + # ## + labels: {} + + ## Annotations to add to the operator deployment + ## + annotations: {} + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + kubeletService: + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md + ## + enabled: true + namespace: kube-system + selector: "" + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## If true, create a serviceMonitor for prometheus operator + ## + selfMonitor: true + + ## Labels for ServiceMonitor + additionalLabels: {} + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: {} + # limits: + # cpu: 200m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + + ## Operator Environment + ## env: + ## VARIABLE: value + env: + GOGC: "30" + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + + ## Container-specific security context configuration + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Enable vertical pod autoscaler support for prometheus-operator + verticalPodAutoscaler: + enabled: false + + # Recommender responsible for generating recommendation for the object. + # List should be empty (then the default recommender will generate the recommendation) + # or contain exactly one recommender. + # recommenders: + # - name: custom-recommender-performance + + # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory + controlledResources: [] + # Specifies which resource values should be controlled: RequestsOnly or RequestsAndLimits. + # controlledValues: RequestsAndLimits + + # Define the max allowed resources for the pod + maxAllowed: {} + # cpu: 200m + # memory: 100Mi + # Define the min allowed resources for the pod + minAllowed: {} + # cpu: 200m + # memory: 100Mi + + updatePolicy: + # Specifies minimal number of replicas which need to be alive for VPA Updater to attempt pod eviction + # minReplicas: 1 + # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates + # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto". + updateMode: Auto + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-prometheus-operator-prometheus-operator + tag: v0.75.1 + sha: "" + pullPolicy: IfNotPresent + + ## Prometheus image to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImage: prometheus/prometheus + + ## Prometheus image registry to use for prometheuses managed by the operator + ## + # prometheusDefaultBaseImageRegistry: quay.io + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: prometheus/alertmanager + + ## Alertmanager image registry to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImageRegistry: quay.io + + ## Prometheus-config-reloader + ## + prometheusConfigReloader: + image: + repository: rancher/mirrored-prometheus-operator-prometheus-config-reloader + tag: v0.75.1 + sha: "" + + # add prometheus config reloader liveness and readiness probe. Default: false + enableProbe: false + + # resource config for prometheusConfigReloader + resources: {} + # requests: + # cpu: 200m + # memory: 50Mi + # limits: + # cpu: 200m + # memory: 50Mi + + ## Thanos side-car image when configured + ## + thanosImage: + repository: rancher/mirrored-thanos-thanos + tag: v0.35.1 + sha: "" + + ## Set a Label Selector to filter watched prometheus and prometheusAgent + ## + prometheusInstanceSelector: "" + + ## Set a Label Selector to filter watched alertmanager + ## + alertmanagerInstanceSelector: "" + + ## Set a Label Selector to filter watched thanosRuler + thanosRulerInstanceSelector: "" + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "type!=kubernetes.io/dockercfg,type!=kubernetes.io/service-account-token,type!=helm.sh/release.v1" + + ## If false then the user will opt out of automounting API credentials. + ## + automountServiceAccountToken: true + + ## Additional volumes + ## + extraVolumes: [] + + ## Additional volume mounts + ## + extraVolumeMounts: [] + +## Deploy a Prometheus instance +## +prometheus: + enabled: true + + ## Toggle prometheus into agent mode + ## Note many of features described below (e.g. rules, query, alerting, remote read, thanos) will not work in agent mode. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/designs/prometheus-agent.md + ## + agentMode: false + + ## Annotations for Prometheus + ## + annotations: {} + + ## Configure network policy for the prometheus + networkPolicy: + enabled: false + + ## Flavor of the network policy to use. + # Can be: + # * kubernetes for networking.k8s.io/v1/NetworkPolicy + # * cilium for cilium.io/v2/CiliumNetworkPolicy + flavor: kubernetes + + # cilium: + # endpointSelector: + # egress: + # ingress: + + # egress: + # - {} + # ingress: + # - {} + # podSelector: + # matchLabels: + # app: prometheus + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + automountServiceAccountToken: true + + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Service dual stack + ## + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## ClusterIP to assign + # Default is to make this a headless service ("None") + clusterIP: "None" + + ## Port to expose on each node, if service type is NodePort + ## + nodePort: 30901 + httpNodePort: 30902 + + # ServiceMonitor to scrape Sidecar metrics + # Needs thanosService to be enabled as well + thanosServiceMonitor: + enabled: false + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + metricRelabelings: [] + + ## relabel configs to apply to samples before ingestion. + relabelings: [] + + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## gRPC port config + portName: grpc + port: 10901 + targetPort: "grpc" + + ## HTTP port config (for metrics) + httpPortName: http + httpPort: 10902 + targetHttpPort: "http" + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + httpNodePort: 30902 + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8081 + + ## Port for Prometheus Reloader to listen on + ## + reloaderWebPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "LoadBalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Additional ports to open for Prometheus service + ## + additionalPorts: [] + # additionalPorts: + # - name: oauth-proxy + # port: 8081 + # targetPort: 8081 + # - name: oauth-metrics + # port: 8082 + # targetPort: 8082 + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + + ## If you want to make sure that connections from a particular client are passed to the same Pod each time + ## Accepts 'ClientIP' or 'None' + ## + sessionAffinity: None + + ## If you want to modify the ClientIP sessionAffinity timeout + ## The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP" + ## + sessionAffinityConfig: + clientIP: + timeoutSeconds: 10800 + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "LoadBalancer" + loadBalancerSourceRanges: [] + + ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints + ## + externalTrafficPolicy: Cluster + + ## Service type + ## + type: ClusterIP + + ## Service dual stack + ## + ipDualStack: + enabled: false + ipFamilies: ["IPv6", "IPv4"] + ipFamilyPolicy: "PreferDualStack" + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + # Ingress exposes thanos sidecar outside the cluster + thanosIngress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. + + ingress: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Redirect ingress to an additional defined port on the service + # servicePort: 8081 + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx + + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] + + serviceMonitor: + ## If true, create a serviceMonitor for prometheus + ## + selfMonitor: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Additional labels + ## + additionalLabels: {} + + ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted. + ## + sampleLimit: 0 + + ## TargetLimit defines a limit on the number of scraped targets that will be accepted. + ## + targetLimit: 0 + + ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelLimit: 0 + + ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelNameLengthLimit: 0 + + ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer. + ## + labelValueLengthLimit: 0 + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## Metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Additional Endpoints + ## + additionalEndpoints: [] + # - port: oauth-metrics + # path: /metrics + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## Statefulset's persistent volume claim retention policy + ## pvcDeleteOnStsDelete and pvcDeleteOnStsScale determine whether + ## statefulset's PVCs are deleted (true) or retained (false) on scaling down + ## and deleting statefulset, respectively. Requires 1.27.0+. + ## Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention + persistentVolumeClaimRetentionPolicy: {} + # whenDeleted: Retain + # whenScaled: Retain + + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + ## AutomountServiceAccountToken indicates whether a service account token should be automatically mounted in the pod, + ## If the field isn’t set, the operator mounts the service account token by default. + ## Warning: be aware that by default, Prometheus requires the service account token for Kubernetes service discovery, + ## It is possible to use strategic merge patch to project the service account token into the ‘prometheus’ container. + automountServiceAccountToken: true + + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Allows setting additional arguments for the Prometheus container + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.Prometheus + additionalArgs: [] + + ## Interval between consecutive scrapes. + ## Defaults to 30s. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/pkg/prometheus/promcfg.go#L180-L183 + ## + scrapeInterval: "30s" + + ## Number of seconds to wait for target to respond before erroring + ## + # scrapeTimeout: "" + + ## List of scrape classes to expose to scraping objects such as + ## PodMonitors, ServiceMonitors, Probes and ScrapeConfigs. + ## + scrapeClasses: [] + # - name: istio-mtls + # default: false + # tlsConfig: + # caFile: /etc/prometheus/secrets/istio.default/root-cert.pem + # certFile: /etc/prometheus/secrets/istio.default/cert-chain.pem + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "30s" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Sets version of Prometheus overriding the Prometheus version as derived + ## from the image tag. Useful in cases where the tag does not follow semver v2. + version: "" + + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig + web: {} + + ## Exemplars related settings that are runtime reloadable. + ## It requires to enable the exemplar storage feature to be effective. + exemplars: "" + ## Maximum number of exemplars stored in memory for all series. + ## If not set, Prometheus uses its default value. + ## A value of zero or less than zero disables the storage. + # maxSize: 100000 + + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prometheus-prometheus + tag: v2.53.1 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## enable --web.enable-remote-write-receiver flag on prometheus-server + ## + enableRemoteWriteReceiver: false + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec + ## + query: {} + + ## If nil, select own namespace. Namespaces to be selected for PrometheusRules discovery. + ruleNamespaceSelector: {} + ## Example which selects PrometheusRules in namespaces with label "prometheus" set to "somelabel" + # ruleNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all PrometheusRules + ## + ruleSelector: {} + ## Example which select all PrometheusRules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all PrometheusRules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## + serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for PodMonitor discovery. + podMonitorNamespaceSelector: {} + ## Example which selects PodMonitor in namespaces with label "prometheus" set to "somelabel" + # podMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.probeSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the probes created + ## + probeSelectorNilUsesHelmValues: true + + ## Probes to be selected for target discovery. + ## If {}, select all Probes + ## + probeSelector: {} + ## Example which selects Probes with label "prometheus" set to "somelabel" + # probeSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for Probe discovery. + probeNamespaceSelector: {} + ## Example which selects Probe in namespaces with label "prometheus" set to "somelabel" + # probeNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## If true, a nil or {} value for prometheus.prometheusSpec.scrapeConfigSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the scrapeConfigs created + ## + scrapeConfigSelectorNilUsesHelmValues: true + + ## scrapeConfigs to be selected for target discovery. + ## If {}, select all scrapeConfigs + ## + scrapeConfigSelector: {} + ## Example which selects scrapeConfigs with label "prometheus" set to "somelabel" + # scrapeConfigSelector: + # matchLabels: + # prometheus: somelabel + + ## If nil, select own namespace. Namespaces to be selected for scrapeConfig discovery. + scrapeConfigNamespaceSelector: {} + ## Example which selects scrapeConfig in namespaces with label "prometheus" set to "somelabel" + # scrapeConfigNamespaceSelector: + # matchLabels: + # prometheus: somelabel + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration + ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb + tsdb: + outOfOrderTimeWindow: 0s + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: true + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## + replicas: 1 + + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + ## additionalRemoteRead is appended to remoteRead + additionalRemoteRead: [] + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + ## additionalRemoteWrite is appended to remoteWrite + additionalRemoteWrite: [] + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 3000Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md + ## + storageSpec: {} + ## Using PersistentVolumeClaim + ## + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## Using tmpfs volume + ## + # emptyDir: + # medium: Memory + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## AdditionalScrapeConfigs can be defined as a list or as a templated string. + ## + ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + # + ## If scrape config contains a repetitive section, you may want to use a template. + ## In the following example, you can see how to define `gce_sd_configs` for multiple zones + # additionalScrapeConfigs: | + # - job_name: "node-exporter" + # gce_sd_configs: + # {{range $zone := .Values.gcp_zones}} + # - project: "project1" + # zone: "{{$zone}}" + # port: 9100 + # {{end}} + # relabel_configs: + # ... + + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + # optional: false + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertRelabelConfigs + additionalAlertRelabelConfigsSecret: {} + # name: + # key: + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec + ## + thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + ## ObjectStorageConfig configures object storage in Thanos. + # objectStorageConfig: + # # use existing secret, if configured, objectStorageConfig.secret will not be used + # existingSecret: {} + # # name: "" + # # key: "" + # # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource, + # # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set + # # https://thanos.io/tip/thanos/storage.md/#s3 + # secret: {} + # # type: S3 + # # config: + # # bucket: "" + # # endpoint: "" + # # region: "" + # # access_key: "" + # # secret_key: "" + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.24.0-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8081 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "http-web" + + ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files + ## on the file system of the Prometheus container e.g. bearer token files. + arbitraryFSAccessThroughSMs: false + + ## OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor + ## or PodMonitor to true, this overrides honor_labels to false. + overrideHonorLabels: false + + ## OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs. + overrideHonorTimestamps: false + + ## When ignoreNamespaceSelectors is set to true, namespaceSelector from all PodMonitor, ServiceMonitor and Probe objects will be ignored, + ## they will only discover targets within the namespace of the PodMonitor, ServiceMonitor and Probe object, + ## and servicemonitors will be installed in the default service namespace. + ## Defaults to false. + ignoreNamespaceSelectors: true + + ## EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. + ## The label value will always be the namespace of the object that is being created. + ## Disabled by default + enforcedNamespaceLabel: "" + + ## PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. + ## Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair + ## Deprecated, use `excludedFromEnforcement` instead + prometheusRulesExcludedFromEnforce: [] + + ## ExcludedFromEnforcement - list of object references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects + ## to be excluded from enforcing a namespace label of origin. + ## Works only if enforcedNamespaceLabel set to true. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#objectreference + excludedFromEnforcement: [] + + ## QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, + ## and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such + ## as /dev/stdout to log querie information to the default Prometheus log stream. This is only available in versions + ## of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/) + queryLogFile: false + + # Use to set global sample_limit for Prometheus. This act as default SampleLimit for ServiceMonitor or/and PodMonitor. + # Set to 'false' to disable global sample_limit. or set to a number to override the default value. + sampleLimit: false + + # EnforcedKeepDroppedTargetsLimit defines on the number of targets dropped by relabeling that will be kept in memory. + # The value overrides any spec.keepDroppedTargets set by ServiceMonitor, PodMonitor, Probe objects unless spec.keepDroppedTargets + # is greater than zero and less than spec.enforcedKeepDroppedTargets. 0 means no limit. + enforcedKeepDroppedTargets: 0 + + ## EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit + ## set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall + ## number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead. + enforcedSampleLimit: false + + ## EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set + ## per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep the overall + ## number of targets under the desired limit. Note that if TargetLimit is lower, that value will be taken instead, except + ## if either value is zero, in which case the non-zero value will be used. If both values are zero, no limit is enforced. + enforcedTargetLimit: false + + + ## Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelLimit: false + + ## Per-scrape limit on length of labels name that will be accepted for a sample. If a label name is longer than this number + ## post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus versions + ## 2.27.0 and newer. + enforcedLabelNameLengthLimit: false + + ## Per-scrape limit on length of labels value that will be accepted for a sample. If a label value is longer than this + ## number post metric-relabeling, the entire scrape will be treated as failed. 0 means no limit. Only valid in Prometheus + ## versions 2.27.0 and newer. + enforcedLabelValueLengthLimit: false + + ## AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental + ## in Prometheus so it may change in any upcoming release. + allowOverlappingBlocks: false + + ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing for it to + ## be considered available. Defaults to 0 (pod will be considered available as soon as it is ready). + minReadySeconds: 0 + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + # Use the host's network namespace if true. Make sure to understand the security implications if you want to enable it. + # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically. + hostNetwork: false + + # HostAlias holds the mapping between IP and hostnames that will be injected + # as an entry in the pod’s hosts file. + hostAliases: [] + # - ip: 10.10.0.100 + # hostnames: + # - a1.app.local + # - b1.app.local + + ## TracingConfig configures tracing in Prometheus. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheustracingconfig + tracingConfig: {} + + ## Additional configuration which is not covered by the properties above. (passed through tpl) + additionalConfig: {} + + ## Additional configuration which is not covered by the properties above. + ## Useful, if you need advanced templating inside alertmanagerSpec. + ## Otherwise, use prometheus.prometheusSpec.additionalConfig (passed through tpl) + additionalConfigString: "" + + ## Defines the maximum time that the `prometheus` container's startup probe + ## will wait before being considered failed. The startup probe will return + ## success after the WAL replay is complete. If set, the value should be + ## greater than 60 (seconds). Otherwise it will be equal to 900 seconds (15 + ## minutes). + maximumStartupDurationSeconds: 0 + + additionalRulesForClusterRole: [] + # - apiGroups: [ "" ] + # resources: + # - nodes/proxy + # verbs: [ "get", "list", "watch" ] + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form

inQ&E1cXrTwQ#dLIJb{sFUgy4ZjeFzbl8d{!Y1;QB)Q5B>;x4+6-i*X;%%R)=tj8L%|w zmCD~jWsaIPx)T+TLE#l{{)f?CeYpJa@%Bn&g4l{S_cFIY@b5l5*!2JHvyTiP=xDAO z${5ajr<}J;s+k=O#&gS%|25u;CVHYeT%*D~5$P$hccwkD&YnYiv0gGS{f{wHuYQ-% z+Owp@bS02tQh^1-5P+VZR$n`t`@gUG3YPJY8|=A*j`=M@T{FuuYtvnkiQ)Hw42thm zEq!7kgghatBhWl^lm2wrcE-rQB@yjgH7!KHIP^gqKfb^OVO3khoei2A5j=xbWxQ&SwiDBuI? zordc*7YZzFfw7u6AGfL_F=DwyR_9FX#xD-%i4${OK{bZH7Nq@I^eHdi(gC3~?bk1V zb6|`au4wclXv!kng2gp=qVxwh3C;_lg!I<|(!5Oq`-sO7;8dxhaNju#viC&g9VqSS zIfj?ZP*Vm9fDc2md6a>X$|<^FESiq8CGe9nUtU#>0J6zJ`gt$m+eGmYp2xxaVt|UBm!=R0j!MC;TeK7g^+c zl4w5M@P=M@cyUjk6M32irZkH4q@=dq3GTxFG@rIqmasR@$6Lm-jCGKw69n_XK$;oQ zJTShY__`9S;g4%11J%qTFhP507F(t2Jtpa2SB`v|?!frg6R(P;W2A%F6mCHxUyOGo zb6Pe@fC`IIw%zK)rv@mijeUI=6nhV} zRu=(5X@H1*WfyzwI7;yq!J6JBl;N2>0K9db3^|=Gz(4n%@JM{9fV35S>C>#7n@|v5 z+v`c&g&YD)w2V4SsL=faG7jS>0`Yojaz%s$P}o;(&WS+E$+<(NtLx{ZdH$pBZ|36M zVgSLc(F=>E+FcE5p$es+~2$<%R%8Q7jQ>pr zS6+Kyb!AKV>doH3{_u(K4Y-L5NHp>jziRr_8U7?JP6?$aJ~lP8E5#+#vSFyXCR%&< z`i?OuyUY1fl~;=DUvo{gc-g-dRQSFgzqvp`p8!PGC!sjWtP^3 zSy$C1kr_={&(f&!vvn@)M`*gXl%~YM5sJ2z-(8gHR_u++4T_M5e8dg%b~<#6ix@7s zoHiLktx4%@i-eIF+|*1TY(+f zA<&kyT0^MUe>hwGpnK!uK&Mk{%s&kSVt{XlERKO9*o$dTN)jX3iz>-0c@J*MZN<^_WHpg3h%Ij8hzoN5jJ+g$2z#Wk4)#q zu`79f2q_&od897=Vx5n?`RaaWA%$UMMC9Jo+o&OnTZ5l$({$e<5KIU?rJW+ogz-|7 z>G%h{cl!kwS-^A zm105Df<6!sAL6tP}?fnG9qp*mJ zlXJjqcrfh?ZH5K^5-pq9b{g~n-MD5Q%J zgZ#d}t)Jl8Tag-b0#~n^6PIo!VcA6XpKvzjKj7?G^IteCljMbr?pGV&wtXZUS%DP! z5*njWsEJ56;3lhGX?-JIZXb{?HT8|(xI$`(=&2-MzIJ$^j8uyjr?0FmRpfgz+rlLZ zaHPqNa)WF;e?*G>{zC6xRGVP{M776e|3bCL-be0Tg&it3ZYRf~$)JB= z;iP!-rAp9>HX zom2PG3$Cg0bA$cN9Zlap{lhfL6kGpL(@xG2yLf$ZlCaW?%lh89XtH*7W>NfRv5~=Z zHKDWNpI!(yeeFMhm{q}FAXf8$88UB4mW+9>uFL8k#-sCWrS|Y3`tnf1r+2FO?Ku#Z zdBVd@ow1r8m19`@;Nf108p%{W@(+mI3Go%Z&J)lIdAr7Qi+M_I&4r-Hpv`bjU}##| z+5ZQ^TG>}S(6}&&<=k~!m;6_Z@l+bEai;Qz%;8w@dVL#Rjj;)lBaT97I?&)q{O<-w zJ&#M%s?UFeR>u$A6jg$+OgMzFT*g%^G`?1y4m>w~A|?0d8%UvjB?v%hCDqNr!uKX~ z&3lwv)g1#ae?Sb$IqtJ(gd0HgfQ)5`;n0oj)D!E0 z-_dUj&MRFx`B+;u7SoVwXGFm(HM0*e{HkyE5(Uqo8-IHDz;qpwli3O?(! z!&WPoN%Cy*RQ)Vrsa@ill+zbX(FisjlhF_C%ph7}=}nwf@L%{U*^Q{!`V{KF;H!=0 zR@aaBSY{}l{O*EBrBR>%%jFRHA6$;&WLFD|2(~a4_6!+}GL~B-$t>~&KCG>JYeE-*BVqCotSNf@j=22dW1!L;3e`V<#*y+!)KU{k+i&wsog=FPzc0j~R(@ zE*;7H*Hmzn91)Rkm8F|zM~~2T?k*jaO7bueM89gPff&od71>G@IVJYzu*+bv7iKX4 z5dzLm2*zwK!MdFyo7S;KG7}*jJu6^|>&lSHqYWA?5-I6AFW!WZRc;^CK)d8;p?w}R zz4C25!&J}-SLSFs0>TNnf^yLXZ$Mf8nD6(5-plAhm--Bn}^7N?%l6JC?m{F`!?7@*9jluY2ghSl2}8%RgEc7+G4y$E)E}^^)ReCR$CkFUOn@e zP0gB=r}UkdjjCIFP9|$iP7R%(n-F9(CF_$o6&i@yIZ?1O&F%7~XSV*n!@e*MpCO`4 zC}1?6d}5$O@Nx6;3b<_)KpfhOfWvU0qfS0*9=v$-Wh8er%b3Fy3RQ=vNidtTA=bYDU5}Q_v1xfjC_@pvgD`;P)^) zO^G^>8u|9a>(Kv+WiTc33VUrU`i4ge!0Mz0>9v<$!Zq_{h3H zR@EZ}*%3^}EwKK=7(XD4G40-f46f2(05jD@!%7KD5$(vtI_Z~ZAk)A)`K5fLp~#@W z+JdP+gc&weN2sSQ7MOy#?waPw^lny?Vp5WlzdB7;;BTM{f2;U`b?}R@1A-U$EIZv@ z<8Nd`{FqfgGX@?^75H^Qp3O1p2ZPWI|6$bs6~Ef&hm9e2PZQho^bZho@DC7^)#CX8 zvGW)b^2y8lX2vX;F7|dWz~X9^d~i22i5&L2?fa(p#UWpLD%WM%NLZHl%bks8;}h0r z7-k^;B)kXfQ4J|vAifkCf=4F0JNwZ!Mt=cb3#1qhRtsz7Tx(HJ?vNr=Ao1-QLO3n- zSF{!-IxWm^^zG{v9b0oR86En82XM?W;Ri{-nC$SeIBmDurEehqxNCu+xIj)0nipT@ zPc1hI?S=!Y2|kgYt`OG@`HN7XUiqV-F+04Vfmmi7-_IN5Eo4b#+j}T(GlT=#a~R}} zxg@9X{bW2zm(n5;JLka&8tdjh#C?^P(4*L5_L*ZlIAy9@>ZP_j@Cech+kRv#mQA+~_(otS; z8HL_zE<-zL80+ArZ-~Y+n8CXsL1PFRK700=!odea1z<;PDE?qbE&9wB0=$5|AZTgw ztfwkcAB`ImXa)hX`?$A;^U;lmLfaqDrv;7LngeX=c*k0+Sy_*t zKxA_rsfZi#`#)12PWx3l%d20GR|_7zDx}c&k*Lj0Pqz&Wrgw<#O{PpbO`j7-0oSi#s~O$@n`Sg2S-kjfz{aKRf6$B!Wo*6M z9>Iu1g`$D&49_c>nvgEZji~=Mxbe*o0jF+-XmHJS2La>k#y`Lc8ut|iOZ;bi!;}`&VdMc%=!K3K zJhoXsrxpS%qo?EfQDg+yDGaEj%0*p*EFs1D1};I|gAE~9e>{*k!U1=p`ypvrPx-l6 zcc5`wZcm<4S^sf4GjlPiMxnYocET`c0ThnHB~8AV3>e=jST6;Q&)xhY;0#^Ggyf|A z zYs8CWR;uDmJK!>bE_e6{6G76hg1*+|EsOhfpOOo30`$5sQTEL>@2r!EK41a~>2#Ps zlbwxM!hrwvLV{?|!u-dCqUeNi7vajpeyVq1z+)E}@JO5uQv4h6xaKNU+B1Fqr<$}G zMLmMXDlD{RNsAK~(XtjcjDylgR_Ia7_e8aMPYVzL*F?CSS>q z8%mm1%oa1{Y2enjjrWsU6o39&Zhy?^?c?WOypJz8BErDA?E#aG@9OEpBd+W<3`Ljs zS5`r+=Wk}*n0S8gSViQxbzel65vBFuTbYBoWShJVH(Yu}CqYO%V}%%#}Qp?O9YFp+qm<+n3*JJB8ih z%=o@fs+Q-*eWw$owk*hz5*iQa_qNPsl{37Bx^Fu zr>p>xw;QvK7A;K4ABH=#bKMPE&bjA9TbhAZ`x;oJVPvq z^0(SeI>@QN@)OLFDwNNP2K!#`6F-{zYe&hc%Z$GyBa{5u7X|lP<0ev~5mOd;Vs_!{ z_ImATUyZSA43@o0I$$UZM8u%kIGG1kiFR7}ic448hK!~4y{wsp?_CF(*2ob*%1>uf zV&lW-sjJy$^iC9gzBMjeeHzxr*HwEDy?h?~QkVQPOAa&mF&mS$z3I%|ctdNc1XFId zGVYNb;e}E+MJhgd$T1;dZr+80k5A`EUUz-Zd7~`NNz>!lo2m}1JgdJQvLeF{_XZdt z32RxD)b%ut{0B&ghvBqA4X{%5__uj+j2mEciCmvyMgesKA$(nmRb@qEba=Ffyp&ZZ zGWBCHh?j5HUM6*z5pK?PWutSYpKkM%{Ielrbn(1gY`vaz)NGKKAgYJNYqPlWYP3>7 z<;#u71D!J3j}i5<%nS}^YkIgiB!C}B!FvM=EEIM@b?T*M`yZi!?{uc^y!@)ht;BOu z!ta5BtSg_w52FdK!v>wURiW!Pm!Pw?DCX*4{h1{G08V~P$tWO3ip6)=+&Ek=nEFud z8a~|aJEsjBb3EQQoruPaC~Te`g4N~NKHQSUuh!@2k;C657yLmUG_elb^3dz1>&+yY znbAmFE56I`FgPiemyi}}_bO9qpMYH8XSkD|W6ca{F6L29Gk$Jg%lo;vP{Q%qQTw1p z-)e#PHraW}19sVp&H8y0wDwf;oWfwe#r}Ci7WwJ=M;ycG3;{3J581{^N4&U{PA6rs%_YQ z_2+^u90jIelQD;&T+n#3U0CWe>V)HFckqR4sbRivS^|R@j$BUv;4jnx0^bV*^ z87j1`-x^A3a@P@(ky==i*E9;HNtbM`>^{HDY04ZP7jb;3Us{NAI)KdMDx|}jLqtLX zcW_+(zO1BVkpL+%X39cjm!iBrajnWtVcqf&+R=eo!C@vY_Tz2_eNGHF+vR1U1Q5R^^6GTrQu5987XsMEEn;+Vw7>xj{u}BEa5Pzu4 z&B=|}k|a6ZE`<}D8Tn?7;cOYT=@8e^1<4LP9WQoWnDRPXmIOwqpE0(j?A=+FgHp%F z4eR#^XMwL*i0~IZodiaL+h5{!aG3h=Vf)7&-;qCS5>w8Ju!7(C1@$^wYU8K8m&M~b z5?;(a`{$|}T+uNDtfnsnTbCO^4(A%qRW0vvB2&H>t&)b}&ljs(Z+fF)Qk^`bQ;e&6 z;5lAT%cu4EM7$bqSmK3c)2FhgI0nEkreIpcXCQpu=~O5ali@~T;aD`rutFgPzfp|% zhV%Ag{5n%`YPu6ShRFVd!o%=rs!FaA*vW=a=61q*Gdso1GRt111Zy{V?W@Pbs>q)pvZh`A zE`9J5T*UfXnzr3F9A{MKJu%sxWLzD}BxBD~3mU7U@6>8latAoZ z#RK<7F;OB{_eStBIK9YYNRW7wSG?OVkiBp6?14gBhB|@%wJ4-dF)O#)NsJPNh0WEk zKZFY{Wnyy0kc@Wn+hj~8ko|pXj<`z&QEbPkTpPVx`h-21ijNoiYDdRjc&@ebYluv| zZjT%)S`)dOY$9tg_jQZpdJ35Gq)@*~WqBX$d%CpwJUjyh;Df!}2+rrT{3!#@q3#E{ z&`m`b1S|*SP9gdw0)6~OVRXRC$n0z}EQq5A4#3f4B{N$$ajj@}R;uA3Lf};j6>LKe z;pv1zpv%M2#9i6v|Do-jqbu##WzpDH$F^-S^vCqBZ{*#e0Gv*w>Ip5?>RXtVps3UsA%73|U&url^AWE$St()(jJhtjiUra-0 zJ-j!aGzoQB|9R`FgVR0FTGYb&f$U$nrSCodj9?-+AIuG54HC!(zmOn_UCu>Q+os7< zw+&%~UCouKi(%uNS_uWWk&evjuJVA{u0>>X^*Q)X1rR4_Zvm(PZa-8+x*p&U_@taQ zb65aV2JjljlKgeLU&EL2#4QemychseLLF0EvgOkg!Y5q92!j1T#(%AE1tuPx30{`N zp>n`&R8ZjGFXC2(vF?!x@+8@c3nAfPcU+UTqPfx53lyz+9Z4nY5mb7+<|h}?4sq{&WlZvLHq#S! ziyTR0OX_FPo;- zy00r48;(V{qZgUD8F$^JSEKF&$P_Zxzp@=fkMc)eB8t z0)!|e)O!K$cEFoqzbD>9$+v{O_6?Zrw%-{EBmyZ#$8&;6deac@b!O46NO(AB|L@6* zZMC>KSB_}Xts~Eb?inUEbTXc`SoA?MF=J4;U7A!!XKS> z*7$ZG_ei$rKeLxR5ZaQRKkTI#7{Fdk1ugs;{OZCux1J>5&|eAiwvjf0%Z$hj0)rGY zant!Xz1)G)f5G_J=gj>H**J5Hu)_W=iViH=>&(oGBxVgj zV%%p7H2)$oZ>pq`{}_pz_e3&-t0^){W6LP*8Pk7Zss`l8p$C50$as%_(Shj?GX(u( zEstAY>*q2COvDk~03>FK2lC5r65}7pC02_!!B?X3ebjY(y2A<0$oZRwJh5fF(>INY z^!)456m+Q$Cu-LHkqoDPz4=#m3qQu29=Zb)Y{%+LmEUxxWb8Yts3(Wu0M+K(<7jVX zXs4kkEYk;y6BysXz!*wV#-CxgyC@h*?IqYWFcPrO3qJNu^Ati9R94awbnf1+)>D%w znLLcH2iM*13o;diUS{!l^e{B>u4w+Hs0%<27MJDT#<9g+@AHYsQp&F3 zBgS%ek114Lf3)LD0%8$cmu9keYeiWq(Ar$gZNEPJFQ#%S8jhi@P0+7HF5Yw=*lY~6 zL0BkG>wyvdHOGD9SBLrHuMYH;k7>K>%#)A5xJq#=x0fv+AyjFquIe&X6V`XDo8d1R z46|bRct(((-8r(CWgZ{8ow6V&SvCA$HA*H!0;$nia8)y3AvlX2a8HNb7|AAgfYL0d zKDh^-@AR3Y65Nw9`RVL0&#=K|8vy}G6-2e_4LGx3X}2oR{xb1Y({G9S^)mK@3SV_> z4V1Hm7+%VElcj#nHm3G$1JH(Gyxm&c<~M=aB6IL?lu19P2qz^bl>en>Ttz9{`e;4l zI6_EygpOQPg8-Lj7VCTQuA6e+DRUm+dy$ZnnmL99aGxyJAJ_dlXsoowXC7yS`Zckh z*ZJ$%T8gvT(K~@nq`|?w99&X5wqIsu>pT}z%_+vqz{zzTXx_AG zHQP}PY!}R4{MGr2ybIo$>YFo=HrX@%=CN6{ zRA0*Z_>cOG7Bxt0Ro~IiJgCX%vUM3$v1^6r7^jX2`W!TJQ=HkVp=eOBWd3?LZBoIoSO{a)b;=&(HNx_{07!PyQW(D-Zd z%zpwZl}~Z`cGxT1{>$wFo}3YCEoZ!&c09{_s)bbL7nxZx`5p7x2(hMOc31HFww@cJ zFN1NBh6&+dueR_PUjwwuj(OMbu+dNXqM#(`vr|c=Re&q}rh%Bgv*}QISy9{00EI8x zkb_;sm9BDajIKY?E#lWSJ?7pYoDCO8>9+=~cq7EV*3KV@${l_p$YW1g+55%9p!1(; z#jN(f(29Yir`mg*f(vr^;IzcEWfycG_a>W+xz*_b^WDU5?pJmeXlKEvrz9)TwPVlr z1bOvUz0Gd^X2i?xE&Q`rCI~csz-pW~q6bAvyOopBr|5z=HN+mqy+_~|fO?)NN$ce7fwXmV43souB273H?5h7h>u~DaF-kMRLI>j!Y)KfLfxW?yEX$`mNP$QjH z=NPYPKtjX}<&~~-%w5Ng>qa)G z@Yme8qG1&qC4lSui;O^mz9uDZ&8?IDXR|oABc0joh$8tw`KS%-)VphKF}S-PpD6Ze z5fK{tR`slx8CObUPa^$DbsL${3&F{7-r?!Tnx3^Vs)=_Qfm3i{ZI9l2_>kuu8;HQh{#FQ3GZM)wf`-Eu zYNHrINk^sY)ga0FRD7W*{K0;)FKMIcL?G|R)2_AO+-eY4yO0DU8!8{)-g>OMmuD)0whd?e~_{#alej_J)XI}pWTunRQ((|{N#AE zESA&f5aWNIh16AWhRwJ~=pCqAEzr$t6HE*;t(zQr3%EqxV*TL6Qw#xAFdWz}?6H~muc`Ewuq z604yzbqnp%8>=2NW;+*&#Z}=MlY5=;($>$v9MJC0+vj%Y^hF`Q=Q!hDaR=4^<#$$= zYSVd(-J}q6+Lra8MUaU6Y~sc=dv%RR&nbu|Fd9qX9%E9ADKm8AK>0WmgO~>5rh)_m zNEA=s{Yex@s0Lbjc2^VXhb~tSLdVcxfjt}kW7L8@e1KLKCrbIWoFI!%CiCg^5enK~ zf~MFi!yO)jA%65cWMB$vJsg#t3?#)DgSwuO(sD6wBQ*0-W<_WJ!BYH>;hDUs^pFX; z(tXnd9m%VdSfCIy`z#PJThWcxDimoZu9|0)L~F|`*Fu3q8z8teVa_)Tqi9DoNx@vv z{Rgwy$!4xC%Z?yB>?&1L>PU@>N}u}8EbgUUxmiLRwEtojx^d+H%`C!GGLVbv0L&u3 zroRusEFAt~77U$#V-^U%0L)@nXP~EYAKg3%J2ZlK-7qxcv*W zKw|$_X0iP*%py6k7v>MMumv!Sr%kc>YP*|zc_($B-!l=Qux3MMkNe_3w4#jczi0(s zPIV%?;Mh;=jp{VFCiMORL@wgD5YO1?UdkYnx%SXbir2qaS3R0v<}aRi*0xbdP;|%_ zInQ~`U*Kjf4gCk#hv(uSL#=VPzb=xF?p0JfU^wE7_wUtJ{WpL7w9=Kac#M{M5}c6b<2?e6Mui&NQwSD*TW)^g_w`lT~yg~}*T^tgxjQ(K} z293ck^FMXeicPsQ=4VRSGhqQE#-HAHl$T1e9iAR2Zbm~LJXSE>yoIoYd<&(UK(cy9 zidTOzt~`JjB5N9)YT0;g1jmjyz3T!rS3&c028;YQ)Y>Bc(@=X$Fbz~7{FA_eJ`1Zd zIGvgOZ;o0=j@Bx5oo^ z9;zl1&@`vAyH1-AW&lJ^PmJpluRAcq#rtVisg(HdW)Jqgzsw%wLq@-6AlHhU|26~p zh5ivF6q7`s0z1~qdM?kH!a9RE#HXAC`B#w@k;-J;@$Zo9Ie&E!N;>T{_3S^tNWKHQT~wgA|_U?e2w}Lb^n+DHJ?;))MdDEf+}Z;StkX0>djsy#SYBQKRCwq`!_K@ zu%Ol>pfhJ|-OJTbLjip(kFdvhba{cjm{1=`h^d}8et4n-cco!uFK4q3`43MsynvuV zy2@H+_NJ82D{oN<{r1YIq)+Q(hHSvL(nZz*pC*9^B~j6;D- z(FJi|^|j&y0|T1}qtMIX5A7^AqC5p&)_;yQzXBjkT9EMS#Os!R`hrc@sP`n2wy;_0 zMU;*e_OfqPwSnlRl>|=tIa=ZVH=FiewAw)$S<&kQPx3u29_9*O+Z2cl5B9GpBmNVZ zKE3?8u_0<0O;jC!6>3=|Pm+oq^yHH1@-fn}V|TynzalMBWQa_i;{~|gp5;@E8^o3W zvs`P}rN}(>cxUpib>!>$K??D=WUE@;@2RO>(h6kh{Sj5^2;S~LTqD;jQK*fm3m@-)0&UQIV{LH{&5 zvaHP@0Uu%|o4I!4{0g7CEMrLp&CQzU>8iWBr^}#y$r$H?6nxdhtya;wh{J;VO1wnD zruxp1u&g+K1YmX9u#6*&fSSU#u&3AC;^A;9@_xii)h8r-z% zN?kE7^BMb&)p-v@K*qjDq>SiP{*NmC|QkC5>{-Jh=$YSXiZW|-&G=B z-izsdP>JB4Vg~0^*HK`GMg#4TgQJ-b7GY(b;${&zp%4P+e8TTHdxdEjv||3OzaU6w zu0u}d5{WgT<4n~jyyoLf71)wMZ6T;bd=M`I?0n%0HJdqCbmREw}Riq(u=_>DJkAz2orHqS_k&(4` zLJW71amPdwvQGtx2dguVD5 zv6MhFw4C;=-H1|={^EIXC8Q4$$KpSP+=Ic~C{tWzEC~Vt`Q!lksxMGy%2UZkmzOZR z2a{Voh)nveyG-kqlK1kpIyp6ibZd!BI(6sG)OmZ=F9-5NpY((dzsjBzPqnXNKKw|Z zXuLpsoAl|lV&F%4D7jk*HI|`!Yu3~Gsn+zNPg;)a4jZ19E~x)#cWYF@eC*zL>eXZI z(WT+hq4V`5J%dY6OKo71tz)G^pkjn=I;-qy=maU!d|zn09}O%mU=?(CBi?Y~`FjPILKM!9LZd}Y9r zf$4He;a7i&J$;U*;bl2_iPU%sUlfQ{StMxhMbw=HmKxjS!B-_f4b@L+oOcGTa{HjS z;ZFD#P9&cB+J&x6$aI#O=U6=)XcSKHshE2)ev30oC3z&4`LC|MY_C!q?#NF1t)Gj) zY93pWLC8J50Bu0pJ3=sc$cTGOofB7BrPuJ;nbWx_d2G)w((Vqb5#5{D!tCW*%u zt^sY(P5hy-TB;;L|J{4U#`YriQ0n~&W5#TLnu|O|t>_2MeV%#*6=LdMcrEs^JB>LT z_jLKRazs{^BmLu}-nb?^PZ{Fy5y3Ylg0N|Rs60m?aicOe#C8jAa1DHi=`=!-fp!g_ z0JDO48f*&nB%YF7{v6`jx)W5YL<+#B9D}gax@fFDi_M%M5m}^D)P8%awhI;j$B7@* zJcq@Qxu9ZPU;eo46@od5_yO!%RX+0Jl~JKQX~2S}%n-4N`oXW5aarQy(km@LWs&Y$ zhH8>OkrFXlhvvC!_%I%Xo1-BLIvoa5D+%)k#|7d7wK4dP;45D**jhQB_ZbR}3VDk6 zF57WQC}(Uw6vyTK$Dk2JM`am*y&)8eNDP{&K9Z2~%Puz-3bcp~CQ4rnW{9c$3?u0+ z7iI{cFApvp(3eL9=*wdy4I%Qtf`k66FOL|QA%*xkp;TyOWu}@m%~Z;5onoYNJAoAX z>*KfdxXdZ6lA9HDtslizS|y$O1vlD>p5HV)qatkN6n%U~Rk#O^9QlobaK|KZTv>;n zDZiHn%Q%%qNE>^mtC6t{(t934zICpw-+zdFD=k0?t9;P~8C63WXilJR84b%#eXKlM zYf=X1G?mWo)HFK~MqlPE9=oiFH!5vs7d>4`JULaTuF{%r(JydHnrf9V0Bcok2t-+@ zyp)-4qu9qfPYdYbTS6>uvvaBL&HdQk7C=P&NC60a5dt49&dSb29>9>4`8&ReRf*kB znnEf%i;?&<6>~6Erx8$Yv%{{`G#Tmk3~6!qm+~9^iAtGJn??&)Q4?kUX;2#mG^h!80UFf6Xrbtx54EXC z(~*@#`gH51t(|&GwF9ZglVL46SIDf?e0$XP=gmuX`hT=f$sX!wN*4p#(}X4f?P;EX z_B7!sKzmvQpgoPy640I|=j`!5_I#IZMPCW5<*Wst#jogtlL0b15k^I5t7KO&cAlWf z@k!?*iS<40-Ief4wHXFxQu-uW3*JIEMooB6S00wyTwy@WwDr8$Z%PG=EP+1EQt(9A z6ZD7su?;kYB2Yxp(2B=#)g;@XNYdb56cNrCoel!4!_@RXTV+sQ>9^g%R&&f zlhQcDnWNalc7$#IXDG)KA!pAQ%{^xjbqojBl*t_q)l=PWCE5PxwL{$xFh-!OH2GPr zT&KMczmxZO4QtKz+%O;CUoGdZMjFk4TfgNEM(+B!;Ie(++aB}7IM3^!33*6~Wv^GD zwzeP;dBt3jAb#)9#HP=4G=hIMS;`W|SJjy~O*Hyx&6wyf#z-`>F9BiBH)7d@>L@(gfP+6ncQ1*oF{#Pi+MGtF?m>Y&q1H74| zfqmWXfCFCx4*Uc-@a{hj`~*1g{>b%rrqPAV;Tj6dD0kuShdrFZ>-4@X6NZ2V0JFgp zn9oQA<5}1ub<{E2vedDAq84;Dv}6pMA$yRQ>M`G_vW}0tY>{5spZ1DF6uZXiJV z!rH0KG^BjG6s1~ktnGo*$%>A~@}&)30wdKZppQ?%jl5zvamS6IpF6hqSh)u3^PXCG zRvUa|S@XstY4t^W8V6D0M9nKr-DG9B*1mH)??n8#+3QX_D7tk$UeWdiKd+F%9}%hjI#~*}2(nOe%$^ zIgoP0A0~II#?o}{-!i#9r_@wvHd~KqOuW?H$lsGLWzMptQY0vB-0YEClCd>M!UUpaD+uChe>5aN)TtAKEkafO{J(aL?B-`@3w-R+-`_ z(Hy)}VL8cn?l{DZV+(HX%+&Q`2i6mOxoC07J`q;FAc?-?Yk#u`1cahuj?{Z@1XA}9 zUVIi)n$N?tnrcVT7pq(aeu@a=;H1uAjFv6ztkYTDkRX(GNTBKB1kWH|h9S0KpOZas z8Nuin^BUzC0!z$IocK4wQ>oKZ?9-(1hA5H^zF-WyX&-+K;ys=_@kRfVvfFUrufO9B;&ry$hsT!GvhOHrQh)J!whM0UvLzA2$ z#>0g)F*1QAX{%+H;!5YRc(RX>RUYrQJE5OtO4Ru^k!ufLY>{!oP!ban#(v)V*_?V5%A1 z3W~$Z{2FxeJ&+U5?QRz@2Dc~7?yH$^w!1&iDp;u}Hx0%9hx+wQJ4Sy8=t zr)?x3w|mPE_MgM!a-iObjdT5Akf7HTZWC|Ia0bA6X2Ul`LUJAgF7vuu3H#z$IhXWph&-?&WPqO#}}eOed_Y z?u&2iGcb-nN}v}qJVOP6@L)(8Qj>$Pv6?PY2WOo0IN|GYR~Sc*={)*0!#^(8HPa-w*ml(9637rn+0kB9wU=l*_awa` zgeB!pQdKI0|0cb0p%J%{CcC#`y5l&yg)4{Q-YJNm<#0wAEi#m0$b20m6}~^f4*o%5 zq6)-#%}JW0By_i1<_OyvhXEwy;VvJDVFbQ{E}^biAsHd++`cb_@6mpl@f_{N;c_yKd&Ou}#)XZGiBCvOUPotn(#& zN(YQrS>hLOt(QGpIKd8i&V>qAy+-ySNM=?tEIxduh&FsCoBP5R*C;i}WFelPEF`Lb zZm>z(tN+Xg7FX9fFmUm#JN6vJ9J*@4Lk5%u79$zJA0y{Uc9}+nv#-^335ctP*{Z^)DP(xF&I-YU>1-)LOWGVN2Lw1y*2TRh2t&D5x4?_tB+ za!bu(4|lz6f2^!K^*vZNd{;kBh$)=o24;YU4R^CzkxXD*ZSq@r&wd_z7vdI2KJ&i_7 zXl?h9qXwc)rYMYU2cfVh4LJc8ZF!VMG&f~{&U#lxx-j+i_glTw_)$S*PCjiCly3I= z1)j@4Bt073$%^s0+OT#U`*_4~BpkGgHFM70>DK**q}ORBtPFX^si zFm4h|2H%Rdq$ZymHi`_EiY;k8ju($MJbW+?TV?||sbADY3O#2qt~9&Gl&oejPG_bF zTZSK%)+4_re=zQZmNkhse@>HG=Ek%-KbUtsrr|afN!mOtAF|M~U_wlCmif7+hr1F@o zfl~&{9WSu!tj1gA0B5WNoN?Qcl~WAYSSL%`8eTYJ&zB9VAX&j0sUAYe8VM~yJ1~Z9 zSr{JVvHW-sWd#-SC!XjNDH<#GhIO;PFg9PJr`rojSO=L5!b~!sNHJ3z^%~1(iUd_x zG|~;_gbj z{ZE)n`Te&Uq`?p|Wcky(Z97M}Z;?jdS!3n|YY8m~Qwn{-vL!%ALLuX1YINKlXbRf- zALhgO#hkjre6x`|#P0wXDVHJNJ0z@AR?!AN*E}X~-5At9ayr|$CZ{RcLGJqftN4Vy zmmjmDlBTMp>d{FspHWl*J1s-L&DnwMO$s}$28z{s0JTNmjkIxv?ou4TZKtvCu03#=WuO^)0snE4oWk(IUvh)>>y zOcim8CfaIK$&GQECM$HuDp8hMu!VGOM8!;iS`>8?z>^1mKRLh>`T2lEptteVP#d0w zZJd?}-spg5#9pvh4xXhh2>WwsKuAhrKlW!ksVw&36q%Y;*XbvchjI2NxEAHPsxOP~ z64PWM=JYY3AK}RqIH~L-Y`ng7WoANdnbhN34xO~3%KDnx2%WwCib=z02%>@qsLxMY!U06?h(^ z?Q7Py1$tzLIInVpqcA(O$9jcI{}+21ArT(0KoDI+f8BnV3ew6XKUJ(kYZIXknaN`nvZSq@VT9+Ellftk3|+c%GSop zy+q_;a*Qd{IA`50g01I^;k1@rzpU0S0;@yBk~zqA%_^OB;EZ8Q?2#aQ)cN(w!%e70D0CQIbTOJrqpaa!a+?ayc# zz7EQkR>BK2=s>V}3Mc5?ej!?A5i;7jAo<60{gFHFP%!|OI zGVwY&64H>XP!_=PF|(N*r9bZ4e?{T3h|7#Uo!8WBC?HCxKf!(sJ+Y8M_bUE|=W<&? zZ<-)b%sIDdx)^f#z>mQL{WVo4Is2A%n?H9PmIMjcQ*X#d5|PMave zE2nQ39uCFXUvn(ycepL(i*NJYHMrH^7>9K1_sxu5~+( zS9^PhDL&lJBX~r}&^+HR0nCo0htT)!Bk-dQ3CJh>&@-_w=e0Lct8qxOw-cusk70xm zT(yGL!~Pm0P+f6MuoHUrYsZZ4%b6yfS4Ewn$x_e~K}~We`N_Fr>x3+$+=su=8vd^^ z@~t7wL5~V+A6R68ki+H#6`%{rke)Dw#7}ChAt(G77(bF=N-&@c>1FuS5-5ue31i>* zt{;(wKURaSXZQ&2QgV9qm)ye9&j+iYm4~jO4;C}PYcrBs{JweNO(30cBE`#N##(#G zV7j_sO!01dtOHl*P#GFPm>L9`iMv+<4cZG*?g+htwK_!Djx*h?e7!BA{0k@7^V9C5 zEjRd8)AEAGXd3>gEnp*4-rnCePe?gJYhmw@Ur6JtJ|#K?sX;wyjv+F9drT9gu5YWT z%(0kee*cQs6umlBVZ!FRAMi-8&UqpDDfD=7E4Rt{dpH{eMZ);i8;Q#o+r)BS0ctJLf6(s~|ppKfF@nL9h6_ z`TloUObmds_J&Fg^iV++bW!}RW48I$RM{omNqGrN;e+Tm` z0gc3_-A7o49LXDP!;T)adtP2!JWmIZI+@KU(~aV9L6tZQ@|7N9eV7PD%^kME;E*KB zOb@kXfu_qsVVcYh-+oUe(S z(f6=01JN3k-3Wb$vCOjAq&@>NomRJxce>&J$7?#zW zw^Wxg=zE#XpW!OWJ*j9D)u{(&pX#B}Lt?4o_(g_kW)YT&Nu!%!lwYA0|6` znhyD?GN?nmn_#pGfzd(QaE8dl^^v$t26F-D5y7{w{xoTCU1K4~b51#jHH*8tD2YtC z*Ux#|c`#CC2QOoqO);vHMJd34CrZ@GLRZf`njc0n+94N*xW53JBI=pSJ7TG9Bp3%M z{VW{msHFIam%pc_BK>+&xA*9o_mNH^>4GJ!5J!Ad#4kmE;@s_=N$DAa7FU#9GV8nU zVIhCR*=O2mDX%RB>kg^$Aa{a_P`GIL_=Q`p9R9IK%v-aJ{qa3RK3Hn!(nb4o;nH1c zCudw~tSCm0HPuQV7*6y2=GB4^yj3)bhK0+e=9TE~d3V|V^af}fE|0|*#CpH6;&5L3 z9mnA(QkM)vWn&M|;mDbO5q+6SMdx9Sc%ooTiijaNXXB zMorQ#B9p@$32sZ3PZwNYpoqzCO{IE}=VGmk=-FA9_C zT;?7A0($%`1+LZNh9~2~e=$20Vs$r!vBVPnARQ2{abAgO*){4XEc+Mi3BR4Tvfccf2j!TMh)9pWYY&C*PwX7M@XXC?f{f)t>NYfr@G3ckG{}W9YROfs^b~mCAXAr?dUQLF-?B1qS2r-yA;l|zglGh(UtwEK{)`n<4{w!C$ZCxSg#yI z$U-*9)DzS(a8oqDQAIw<5ibT$z|OG>N`-k5_DN6+Qk>sXy|a{Ke5rS!gyII8tI8hC zmt#6M2t?p7J28ZpfE{7QjdEUeQ0<#8f|a;u+8`I2iG`2?ya=gfOacrxshCz`0IBnK z{OuuJoGGM5jEgzpaeiqanqEFsl|ULYxaTqeEb@6)ifvB+yx2hUNsjml!`qM zoeD2}1%rd)7t#NvS}tA=Zi-zq*kavZihMdNR{o-CA1_&&AS^-!U%;9G6Bwebq>h~6 zw3X;h$p*ylmCat-`g|bLTZ89UVE6h|R%38Gl4E;2PRlLcSIBd3gCdE2?x7`DoD)xN zF*&yX_m483KZn5DccLlJqGCG*cCY&@@xciDUC41Xz`1Eq?DY&#^cWPO&a@TIPV^ z$Ei*5ObxEq1(ct)54`1e>NRQ;b5%Atk(X_ z=kFuDtDPScaR{8fN>fTMi2UiClQ#~4tp;z&DcF(NB=f!lN3My}psp@}_w_xYhR^jb zyR1;1itJk_bECN#*|`04XFoz}e$k32o4UripdOoR)T@YV{Yeemt|AuC(SeTMFy&Xp zoOL!kJ3$@!wukf{EYfP*9IQKAUzm=81mVvf2Ev{(O6=rF)XQsu2v4uz#-A|t3aNY? zNfR<%2F2jw@_nz}4i5^!k!rfn&tdTz`TvN&6{ZSH&^6QmC|l%Oo;>hJ_;tkUXtgsC z8LCth2W;2ChYQ`=v};OxZ{j$+ZwqDc?Ca~A7{0I2kpVeN=4rO&x4dncOR~6{B{IQ-{QOZM8XK=$*6Y2!Y`gkNmZ9PLF*sw?7w73_Qm}Z z>Fxj;=cm>kkwSqDu`Yp-^7fgE>#iCB`tN7v7Rh5~*(W$m0rv_5UJ^AevgqIHFYFVr z{sr{@+o)`lSt_bf&@! z@p}ney`X(eSrqYV)TSqf;a7< zchcQpecr%-Lg;)>$jd*OrkD5!*o_h+!TgT@-(dIbA7HmxiN2}J3wYPpe&mFVr+x$s zb@7lZKT(}ufgH}~mT8P2t-w;bTroGNvc^&jB7*9d-X{Z*_hSsnq#;cRIb2P+?q!}3H5C@k=Ap!3w|Fg| z-Aq9E)_bB@LE0)o4wNBK3HKgG4dK*v6Q^(wv;W@UpZ%=8Z&Fql3 z&d5DVOD%G6gi#NZI$7xV_6T2h9kRn`>J;xhaAVrB+Wql6^`}dq$2nc= z0PTXY_<{Sj*X%F6#k9pqu5u;EdW5?Q#pgWv5n>$X4miD4m=}ocEdBI=_O4f3CW`)f zsSb|j(>9L6b1&Q)4uhy4+gS$ij%aTOWwGM4Fsko78NY}P<#E7fF3sM#qKBwmt!WLL zdM2r%Ly`F8&Nv>pN%|y6Wxk$t+|&k@!+NEofU*wRH~p4z&cC>q1Q)3ht7_bGl{ z8${5gfqZeKx+u{NDPw&9evE%LqpOuA`s1Lqb+5<%C%BHrb`a@OB+UG;{x8ziqTfpockIw&r)AFrtBU06?nSs5C3_D?sf!ndc{E03)ZPmg2QKV9Hbo%d6b zD*-l7GSvY(1o29rz;Ypv-Ar5uKoa*ZCaA*)e}cNR%okq9wF4F4`B{uW%@+*SRA0Ky7k4re!afr$S>VK!@=wT9a`@}t_cVsotpb~~ zy{r%>&u1*7jnWUC1fHv2nuY^@v_MPAjFcMOM^UUb1-bfSsB?;eFWIE%*74ce1-1sp zpD0>EmIrC5SA_benFIOul>ZLwmApYzw`+Z4rSI(5{JeMru(f>~!6_kPqk9Y_JmmAx zA@^_}9sZtCpAf%CWgzd$ju1W@?%wQSN&9=6fY{b~5#m;zDU0owe8Ra1fXM15f!%X9Sos%pv1j;^izLTmAlmVLGQNp-HaTC@!IC`!!!M@6pIr zHAtQT3D)WL?y-)+j9e0Ung1%K!#{=%CQM_ z*NzN-c2H7m4skt2F7{lxoLyY{UOq_^hs07WkQg+_g3~u30>bB~l!5b%RS2XK%bheq z^{i(|8#e@GPT}@DPlW~L=xYlN_6A9X`IIL_h4~V@Mfp9P_GFBG*#b+#C)}br&k*4_ zuX&aPx1mOA?0T`lMvJLZ2%l4`PuLdNy5e$s9;hJ!B@%#zod{-&@yBcu8+@9Sb08DZ zBH-EY8LXji1mGN&x>;m%&HCsUSc|aGO;HYa$T}mZ{E0%VBLX;@TIx9PG;HN?fgy=R zH6e*)`3ex~5iBQAI;oWe1XA{I#%`cr(y*-pDB#!>_}^i?8DfO7Z9hY?t*8dS@{8$% z@zw?pA5CYHo+8Y1-ePT!Gq;IHv9L|R1rDYV)eNTmem(PvV^evodynpwmu6*?VOld2 z*!|=mtYPCQR!D#ui>Nw^zypcQdJ(flpDEdXX~+mqggz6mvp?>WLO<{F!Up4etif1> zR!l|_3=3yE!x%EjQY&Ks=gt47X2-4nR9L(Lc=p|PRSB(vx9(md& zTJ%W7z#J}^pb(`5vHtWD2s&iymj=VT9NP~T+zR87`0ZG6&}CI6)F|)jiY))e;?%`V z7S2GhMo?Tk;f2Oru8TivwJ7-ivsx{JdA(5ScLDtvJb=L_rdmN<6~+TF{bsR=6;L=? z*8^_sW!XglAIn1B(Ky|}COBqMYF8h;^Yq)61AWU5fzm7tg*2%l&b{Uz7YoAZ1>IjR z7NLsO1Y5xj$28rw1r{e%7G~6Vn;8399R}KF zx~Ah)sr6u?BeSv}5^E4e+Z=g}GM~~8g&|YMIB`3d)`Z~sqGGLvjP6=TkNml{J^Bs5 z{Wo|&Cj1M$H?jPe<~;gGa|U&tpJTf~oS#mje)v0fPk<*C^Yt?QjoI-@2rp$M5t*%e z%MZPv*##k?pySDJ2cQ!j8ISUQ^$ZKI|GMs(p2Br2de7wI<4sO=;N-zCjMweQEdGZi zz1>LjY<(DvfQX7o3YbA9iTvr3y$n)B@yLjY;vTtZowt4r$pG5W?u^oW#pZT1!}=HV z#zyuco}0nHm^b?D?4LkfF7rUWqeUKa+uXFC2W_RG4h#?PC%k@l^lK4nHg%V|1Au%O z_+cf-g$ri8%_Nh-lgg&Zcs`)8V3pjJ8+yi8iDNjjR=!%!^8XIV_o$Bt`~mX4`u{&F z)9H18L-G)Rki3yrG0krzkGAqRBrnJuDti9NSr|?FOIdzei3Dn_mjx&s+>J^{n~c~c zJ^G(0Ug+OYe9nJD@iI$om}3&>D{A&Jlg3o&>{CGJ9jmK&M?2N^OwC8<=_bZeJKa4lo`WtnR@s-G;oilbn z0`VfF;==W3EpgqNvv`AlOCpj93R0wJE#GiIJb-X`u|HaJog@FdeIpy!o)+V(deY!; z_Ko$yf31a+hmAaN7p1Y0-#;*=Hi-i0FZldl0l}D zZ}_CXiZpnRIcH7~ixmwmS3bBJJxB1pbZZVY_Jlu4;PjXo?=>c0bn=vLwxPMgJGS{2 zc>~oea_XV8xR3Jgly+g*oLoa#S+q!$z+-gu&zzHLGLgXpmF86f+>1c{4|f7M0Nh;V z=dNVF=J;g;AL;-Ui8wZ9gQ(6z{!IZ}3TRlG@^C7P!5S`U7}p)$)w4>+jBd*i1z96D z1TxrF7Ts9ndscy_Rq_dm>0wyK{+%<6cw+Q=A+vo^v$D)0>+C(~xh4i}n5TsEkTRbvxmsVp2b z5-a_hAayT>f2UqfO@IFc_k${G)>J^LHxKkt=`nN$4Q_r!Ly>J*9d2LZ$ZpVqV&*;x zr*ZIZdij|lqp4xoGW{9vO38vu`$T>*_-rkyF$=xcKEzgMjTq_wfX6@n4UbcHpjDDO z4=g+ct#pc%YPvX!*ARh4=6rPXh@27kv>+=|!KNI#`g~!FTmds;z_a8EW3ll>{d6;s zRwYjH_CDWS;mo5#Nbe#2(M?xkpr>55NwDzwDRvp+heGm^;*Q26!Z#OA^42jTLSkf8~0nw(529>WSS41W-#ni+UL2BXzWr% zqpu@Yw_@{{-ciDQp-Dg8y*CbdbyburG#>csylqI7Z)Iikrv`N~rZ5P+7%69!KOIPn z40ibcFUH&JTvb+Gxbh=-#@!+ zS9NvwuCCsFt+jsFwK7p(7sS*poA)$Iwqp;@wAG)BpLy@p&VbI;bhJUDTV|{(izHu6 zBf^U@NM&E>VUXbjiAr8?fzwGz%=36kVeW$qImAi5(cvUTe+&^Vgk{U(#%0rf*DRi| zn%~dwA`UG?e9q#XjodREuX=}RcV$Q+T526s*~_sDRdS=!q2p79FzVoH_Nq54^rCSB zFLJ5Xe5MRo=?)pveAX0vGj^_Bj-_8!o2a=*a$=~`!CSW7WSZKnO}80P38vJVmesWD z#FScAI<@>-(A?IRsHE3|b*VjOuDKPvx(&*P|1}>y>>3uG06d z!zq8K8a6A<{Uc`a4zdlZW9f+7)qK3uL2R@hZ{M2XKQ}^G#?DuH9Su+;8AG^v87}#; zp-V}|4y<@+*0&t3cZ*WO&RaLZB7#S2xli?Ucw|Xljj&Jqf5AWG?eXvSj^7etp6-30 z)%|nEaPxgO|9w`r$IqvPbG7?H0bqOtman>15zw+zjqkfup|zd8;rYC7 z4h`SbB5wuV{LcZJ(Xxt$+Y@SIz*33?>{mA(bg`g46fFYu-N(Na>@TVRsbEti1%^|n zlDtjN-6GBhH{_UqHxLwkYG9FBL}`h>nA6vU7-I>yeP90mX0f_WR?=BEbv>)>!Wp;- zxZt+^%a3K)w}!eN>Z)W|_yX(G_)-J!&WLOaoJ~xv=uOb&%&DA^?*iG=* z<{E)>%sZRqj*z>6gCxyWd2z?uj*y1LF`H!;^F?r*jl5>qH$r7vW_i}OsEo0DQW!9$ z*qiV`?1>t(MXQ)DFQ=St9?RcdXehAnQsRR+f5iF~F*nU1Sz6SBy+)n{r)-Fbt0g`V z5cHnQVKywk+W9AqU};#ImgW?NCIg~!$eK{ZmJxfHb@+JGdsewNDhkbE+4_+~GRbm% z!Ftfb2&37Vf-t{#Es16&55>L-Yq*t-WX%4wN^uPZ&&0G25eS@)QxBX*S|9^uJ#l6% zje6tOzkK)nA$TBu6XbCCo|elT_bprT%b}iaQo4S+8$&zH67hB{`A`P3Dyk~hSQd2^ zo}vyip^fgky8mz;ck@ zldRjUtsO|d7Ul3!T&NZ>{W+DCu9S=745N$F>qvlt%p$TY=%W9HG=@r@aht{ReP8@j z1@kFJ6{S#4Mqh??V>v#cQ0LFk^BSfTO6q>pjeIc_Fs)esLXVzN$Zw7=q9Ht$ZaIgP zU!VsqinsB^H#kq3!i|Az!5k-(%rZX)q4kbo9$DdW$q~pTuFuuCPikyKm2sfEd216R zWIUI%=?rzrN@#CM1iiK6YG(`7!1Ke_?!7;kIEJn#8b#9pmAtzI21luJ`i(d4xLQo7IKse$=LA2Q+Gt2k< zwK@DX`MdGl)%tfD<(rn=6rC<&9r#z{*~t7q8qXU4YCKyH<$))Th&PAQd+L}x@Z(f- zv(t-Cto@rpu55x5)x+3qY`{v|z4HmjVV01adbF&n+3@eM{O@oH`DSoN;@-V~G03qD z(a$7B5E#RHE3Mr@@Ohwy>xra~!|RFjVi1d<>}hbNDJ5Hzf#T zkLhy=ec_MMf|zD-?9hkM)EBkE+KocT|Nj#|cFg$2%kbJ7LV0pbhdjeY{5q zyg@b>tYmEz6Tb?l8W30Zi97l*57n5YIBggG{m_|;3wk8MI=)V2F&_i`mH|y ze8Uc+kQ0b@v)-G()WN&9Kt)|GG5&b{E((K$1K*ds$a-zW=m8O(j2jjdBuArE`wyzZ zGwDC63jA2MUsir&nEy#t^!#5`#r}U%6-qG)6{DHif5@D->s%c#sW2z((fcoLsHok^ zqLG%vjnyE=8v+sPO>8;7)vEfdm(&W=di}rEsz75CxCh}@^tW0i|5mH+hCVF!OQWm+ zeo1*VRj_$W|L2BH{z{b}fFfebReI`*r~nviMZ$}{pcy~j;LdHIx$DO#0YByqh;BjR z?{>^i3F#~Vud5mHAQVc@>}lGNDJ>STEFg|{s|dPAa5r{MVVc&X@&>7Fp=k8}lIenKTxE1pOFng> z=SU1msTeM3d8|~hSk;)MZP~zWS%7Pa`LsF;P@3FPSPu>y8b-EAlpf=u!X~^cT<@h1Ni8smK3mDQ zTY2+y@)XA7c6hX@H`@Tw`#j!2kOQ6nY~o9xRJFrg#h+NnZsM?L3?z3kn?o)2-A{C| zaJu=}2MC2o{LVkSwIkIJ>ex%Z*HjeUWD9d6ZtE?rOIVk%D^X) z7{$cpW_v0oOO?_}CPP|-?rt7juN-}A&fvZI_~Jg_c_Eos(=SL$xR1SGy|hO)C3$IMM6#BAvkJgHsZ zY6~#U*|)}&WwXcWeo5rnV>UpJs~hzsc{Hw5E-nhsFJJ4Qcq*9!uy`!qOE(N9b2o_G z3@>9C#5sZOBe9r#mLF}Lz8^?4L^T9OA#o1wEol-$W6tV$f+ezz&$Q0J4LaE+}DG5PP(RoWK2JaXYAen}l1IknCeNZTUh;fv! zLU1Dw3NhDL79owts9Zo99+g}xl;u+62x^xohL+&(r)-ji+sR^OjyS{9s@jc8H)GPvBwrpJGM1ZrRIMIg-g;Fnq)~h|1fqQ8RB&gu?U+EtbqE+e zmOrca^jg_&H|fb~)+{ErR;^Z)XsXdYl0R2z(iQpVKtjCYnT)h>Qld8BLaJG>m?lo8 zy5zzA+pu^mg=AdhTor1p$@cpJ=`2(g+V^I?I?e)3?+jP zhk}7%xi^T>c3?K?6@H*K*SM!ct+GdrpYhF0ee&sB!|^$ zPM!)kyVEYx+yB>A?&$i`!M(MthdKY+0LAP_1;2~5@@lt;ABK@w{vD?1_x8~4P=(Pr z>*Ppum$KG3Qc;{YuFi=11)Fd68!MQ7j)dygK}qkIVawD?B1tApl+B+dsh94aM}=H$ zq)}rf9?z43h3o>CtV3QGtSZVaYa-0u5qmHp%tIoQ72QLx02mN!bRUjIxR<2xdV}8ZkwxRBq=!WO)W}B`DSv^Ni2!f?%zT`+4iiNTauY)VolI8oGKZoX+))9L#6ICNzhIn8Spt z-jfgNY<-`ngsa|Hmn~Ak{(i-YFEJ~Qq#(`7d??LH1%aw&Y}D`mezr_A_BV5F0`c%| z{A9HQZz?>E))#BkqT&7@@-!zXiH`1@ZyYqsvf2Qgn7EyHiVWG3-j_BSyE(4hS{ zvLDj!AK6c#svnV51}>?rhpiKi@e-~8aSL`QHEkQ5H51P^-wzor7@n`h+#jtpM!-&` zWGHeTr%-;y_$~Wg)AN;pT&4?*KtKy4IQYI)4;!6B`^)&##5KeT5gMex+!MvR>kc0t_J zR*8~%E26&=mc}<$?d1d-cS&IHwF&=4EMP1!PEh7$rRI@Zt*Z;8P#IxOI08fU9pO%K z#ue{Xx{bbIz*tC70K2b=8;ad5=>E3c7H#lXmxT`Pm9y&b$R1jUd6pwf3hqU0DP=$c z5pq#f2c@WDhBM2qfe$*&8{Q{grR>tN%~KHb5}=@r`@m7{$Z_e!$xu#La+y~a36{K7 za`>woMx#(f@NThrG+xOnS$uJ~+WA|IBV*edG$0=^yfwFP%P}(rFMj`BPlbJla%5qp zSUPlHOVrq0-q;96rzCbf!q#hTIDNvQ)ACO0(&pZ18)btqHqkV)=D ze;o2$y%%`yCjRz5RhV<6l`A^Z7UD1?SS?wV<<}G2FCNP+x}f4#@0`xAvdJ_?VKRCz z-hVr|M_^Pkx+N|p!)FqE#D6*cj;|*|WeNWj#_=yC1WFE74+*ul*ioLPrN-Hv zy3{WqCW{a=1kZI%O<5R%%m6!4gMS${KOO(msCmiq-x@V#&i}uRnvSRcY1GWy{Ku$S z+4^szX6#w-KSoWQO!c=9nig;1Z=)vnP`@!8=O4Yv48=h-G`pz5uX$%zck_(yoqqr2 zTv*S*e&-cD99Z}mICyEM>_bAoHr|lnZKGOZ+PI=Bd<10j<8xu2=r0^ zsSTvEaMLrDXMVd8?9t|TWprG6oo=H(v6J5G?HdW(dv${9U}I)R_lww_U?W?>7$*GF zqrT`J;o{HYgR;sv+c!LhjR4}msO+jwqo_Pi;59`J{RsD0mfFI$m{f$B1K$NE{Y8ik zWAggF%-oi$*{4It}FlID)R(K3bj`#9uFaZ8sCkZ9yYKHeO&a%#Q3sLcX=DzxV20zL!da{$kXAik&nRq9REMQ{8` zf+}JDf7M{&%K9Pcqu*GvduFf87Ew|TzRODt?bSGeQ_{U$JK~+oi8|3@2bDu5D(yp z?PJunyy*`i62EPkt>{oICPM}0Lo!q!PPCpAGew+ycZb)m);05tI=x3;_A#2cj1=`< zDP2MuSp)x!H)+0k6lIx_u7(96!fT|CW#zq?q9cMGq-)z$;&9<9q;RR~6n2%co*#i= zIu``4{_4=W{MLit_S4>(C!G}AGbSW@LyYNvkL@SIxHC3Y63tILBG$@kQFFOvF|vq;L)W&XQFkLUJY00dk=zU{ zNLlxTcR_6pmRdmJ5}m5zn6F0S1f9LC+mo)Ur&5)-7!}OPQALYKk~;jO3a$T2p*w_! zi_mpHDYnc*q-rX72huzh!nv&anQQD5qi@)OuD@#JVCVdF&;UF1PiFEfdP0Ewk>bj(v?ICHJ9v?`|a5Lh}U(&o!{FYRRt}AzR4J9wUZ)aMMSA1Kzn{ zH?_QoWpRMOtGvuaKw&@sB)|(YKdg{CM=_X4N9Go>LnPtL>;iS{73wzrd&P@CjHE;) zN;%MxRixKcd?NYLc@t{_hP1h$UI#{iUS8nSeZ2lhiR%o|cT@{gBF6?Rh0qeR=Tc-& z+(|=bBStvk?P<><&5yp_}ER0-U_!{x6aj}7bXQ zseuXEWCdE*N4?q0hmzrCTmxpL!F813Unk{*0jZaKA)%i_-GI*6I**Usy;v{NuN-o% zLI9U2Al+714HUtZiGR0`_RlKqn^z<6?A<0VJvVL`1JT9x54s}D?jkky?!MIFh#-4B z)lQ^JayG`~b*jo~&nG_8mip@;vC4(6vKAWshUueeme=e&Q)_bp)J$d7#|_t0Ie zvY+-%RffB956q-5F3N}+<7NDXwIn;zw~aI_tW0lV-P+w>-AiUI4@HJvz~`^T!amc+ z6Okb#-j3P+7nsQb%tjezsXg{xdQrY;y?RBeO3e;KK==_65ZvMM{&wF~+%|-@!Ys0a5)28ufsxL5l zv%&!vkjfiiM?Xyq@LqT&U;BNv8GF5Rh$NYUjW&!wj-D91t0Lv*PM{}l@IVrqW@Cf; zV)A#xjf-cOXzTv!nR)DP-F0!+K+;45V0x(QAhd_Dsa08kFpx}_(|~&T@l&bv$BMc9 z;COX<4S!ByWXZ;ZnMQHuE&b;FJXtDPi~$YAy;5I#z!4^9y~v~b$A!z;kKW8ZDCa;pB6IFnVy9DrfKO4YmDZ<}&_%8b( zNV>V}YHQ0`+Yh66`Q<^6j(r&osttkuc5DCo{&zP^ zaBWJqDA^;$*A3%3@lLSY3TKnu$qHh3+?Gxm4S;9sZpcB&2@Oc)u6J`f7x3g2y76_b zE7<-qi^FvbM!Zxhvz7vLA3hIlP2g%Ml)t-a(X5%yB9t&L6~&q%5*fGBiALW|+ib2^{Zd*c$K>1VJ2(Lzgt4tLcXuB1nRDsH0)j&X!sF?yb2>!8*wGl8TsM} zNG;pH8C6gUdBt+kGxjR;$g#X84%=;fIaO*7x-q-2iqGB}@aazX;de0L!P&+TeJU?D z%6?#zChAP3$1-MPC1k(a&Y|s;qBJs-4^5KHqNVJJD$~g-)JRm=2d8ev!rwVr0u;LL zMm3Ua-s&w>+Sf}78@o+}>ZacCt|J+dO+YiVNAs1duiUj<%1aWu?8Q(u)#3-DZ#Xar%^GWyZUy-iX24Q2GNjV4Z3sSgdJ|S-*9ZIa&a_V6yLTfYtp+aS9@00syvD*TxTy3bJ zl+1C94Pl;e&-`MP<_vOT&TVnVG!vSuIXVPF*i9!)85gWll+rkqVMwKXmg=r3f})^i zSq+?@RSoe{>Q!8}wD|J4`Y23t5$RH*y=}vB{Mjl5qiaWSoL+25r-`vRTng|#-S%-Y z4+Hf4{Jhh6(RGobR}Tijd4@_o^F6%Ieg-Aq=5kxS#0mMwN7R9Um?BsH&I3lWIXJ$_ zp>8!Xm@PsJ(;+J!qaz8U(>iLhj(99MH)?l&Mg?v^>Y)mDa{FR<3gM z_4GO!Y?)TpZl-dz4kSJmQBxxC!0~7V9tC@w8;LGd6D_m$hqw+25PkOr0Ed9s08Dm& ze`-BuM8tP&M&U<;{D3x6eeIb%Y-T90!+@^47b#QnQVKb(c4Zt8S~vg9{y>A`K`(I@ ztfwCWnX$}zar3Ug*MX@8Uf$n!kXzML2D(T}e;|?bfo1nag;Qy} zqbn#?X_%|wT3r#G;+#Ssn+1Lib@7!gru}4S4)#w6GB5~N+Bq`V<^adBNdIQDI1nO3Kx9RbsFPe z*uRq&ZOrAnnlvDOVxE_AKixRgb$T%tC{7)^m3dIt}2x!twQ*)=Zf^ZVSRzXq7h;rxBZyCz9G=!Jdp zld2j`_32~jw9c6B{o`G_=CyU;u=PZKRelu1PyxZ`INN+^QPmUJM#?)v$s~mxLemLd zsaMPyGY}vn#*cl`GC-*LMFZ@dO;D3?S#};oGh1N(iOg{$b-qA&T+MAemCxOsv{lHm7T_dZBiPz3L+7tt z1sMDyzXlZYVVVQ%o{^F~IseMO5ILhxbS!x%a1(541ZUkIUP7<5O{$%L>29qGthNNo z;sjpP%wzc1iCOP6T0*s&}^S z4I$xJ6R^Rps`U$Big4ylS2*UUU8R!ls|0i%t_dJ#X3T-p+$gz}CxSNSA0--E_+TD-nIh2NV~}{ z^ZRi5f^cQ3Ch{Bi&SX!MwiZ722KGE@wi8^jql~?*vDCgL;ukMG6iiH z+B*n+nhw*+jqI~5>4P94Gdros-trMgI0TnfICD*#5$ZQ{OZ_oPa<9Eji{D9WSrg2Y zuL=r;jyKEO1q%4v4Cnj4!K&!_fO&oZL8I?s?@wXY82Ofxn#zgzY(VY{rsj3Wb z|E5=Sz((&}nEAf=+B1vbG$>uXTs}Eee3LUC3}o=Jl$ABwxuV!}@=a9pP*!Y=Aisk5 z3fGORydSvlzBCb|g~Je8QaUMTWqnp+s}g#wue4R~i}Us2bz2Va3(XxA=76p?>DK1^ z$au7eu@)_?6UT()wgrGU@BP}?`RpC(_1cx0egEp@0^r);K6B*wxQ%~svHo>iG}}G# z^M0m1?)UTacH7!>{l~>Q9SU{q~0g#@K(cVR-k%T5IE6$ zcmdaEV?PxeMMTA(AEE4JP|-v=#A~&<>WW&FE*nXm)teQ-d)emq zzu_e?KLaZ@e?7lXUh_RXyzYMA#8&}JUd~Hz=nbO2N3)t=#XD*?Dj) zDkH8*>@L4#OAAOOz7(RcUq9T2rxWor-Q(`XoGEN2Vm=j{b!63FGweL+5+=r}X7S64 zrK;)TrV-^vSg#fzLcRS=kyKXbD5sHY!^7sKRd~XaaH{eY@f!Yq4hIHpocj?Luw4K= zm@#pWuK~{x8($~Bf(%vT+jC|9K4;Xk7;ZD;QZ061n34y1npQr%nN27f5)$&(*f%K3 zKj7VJFl-R>X3K?q1{&cJyz@`OP(a=;e6ky;`Jc-3bOe)e(G6?1K|7O{DHcVej2Puw zuoWm!b~Dz3CX1-)nAJ|<;}(O)`Bt#2;HXq4i1ai+8IRfY zVO3l$G)u`&6H-~4U@hK33f&A8TqC@@DDNUx>`y=emfAUXyBmP|f&ef~FP0nsJr{-ID!Flz zp;pcKlzfEM`l4!VT)bB^os@+kHEZujn$d12QYoMWa=AWSbck7PQN z)KG{Ob5tkTt^>LT4|9FSMm`UOfRPSj$cgvspAX8~EW2d#s%3RBZ+-0F*rY%=8qeW*`lG-hP9uQ#5WU04V?tkSxH%%>x-1S>{rGrq`9`@mzojAE0( zPPEdUAz>8DxX@q>oUX!FP7KEq;{3(b!T*H5B!x9UTJDE*d=Hy2`NP1|;PI1*C10oU zsvM;}zgDro7_Fc+C2)LG4iB@MSD`>9HA{*)*`-Z*(e!;0--Mu(MM-EyV}8im?6v~= ztIFx9p1#sBwh-10osVG5C#37aT+|_bHQVv9nVY#h!hCp#)lp9`h01D3>4W7spS|MO zjC27KlL8o=-Xcy%X_aI(Z?cFFK6X8{d1++q{#is~ag=K9OzI7%R0XSna^XZ#L;K{q zBe0DgwxqF_14mCym4ay-rtWRo>{M9we(tofT!K{cj|#Hi82e#2UA#?IA91|C+SLYR zLF2PGc(0g06lAOj8Yp~^t5vhx%z~)|Zq$}9ZAA82A76LP{LFqC0;d9?kt20QHO-b= zGpehmIu{BEtwT2KgnU&3pCDbffUd3}YC41ej5-s$1Cue*y2NbNm_o z+Q6!QW=C~;3k9WcdXHHaxs?7DaH+?0_=&n6GXzI`;>Y0|OiIP57t55&M7=Mm z^2s(DAF;KH_Zvi(nn2x3Ji}36gGbetyKvFlX;Gn)nLhC-<056IGH$T}bq7Dugi*IL zGE!aY)n-}NZc6@aVHtM2%p}RV`39q(^W@H36 z0kbr{T={`{)Yw~b&IShB@xwMq7eOE1n@d9fijSrMZWS~ITG?fj+79aeQa zvLY1feN7`cUouOZnw;+=J6!E@i{n`-Rm}V`Uvz-Iq!X z8po)Kw(8lCpa)I>*wqoSd^Ul&JQ0vt;w5=9ibXWK@xKU!R#;_tky9MrwM~8{`4Q|< z<$G|}dLe4sEbw$1r3TwTg$+_eKau)#7R?*RbKCX|DFQW?z);o``znG$a;+eOfpr%1 zc?w&~YdA0SGLL?KjnJzYWtx*zs@ysQs?~Yn5k81;h0_TmUUI+49SLlunah!Rm>bDX?&UzIv%pNvAnkMVAoEJBPQuCPep9QRKL3^eH0+?OrJ8c$5mU3OYP!@(ma(N? zhj@T@B-0S)v?5K1m52uoW{|k8VG%rNum;2@R2r1J*m}h}K9B0tM!5LvxP$fl*NJyd z)vEy?bsn1a57tE-a2FDZE{zC6*0m6Fsku4xVu!!E@oR{vV^(0}BSoyvn$pMG!zN_&YCb0r?L z4i$~)pdl!|eB4@XJ#0iAylk9pN@67)!80rNB9L}zBYOiZI<$x0F(=-3R3 z#7*h6$+OkjUVBV?%J|v^L#NF=YL65Nb6 zirUbK?S~b*zS9O_a%K(e#hN-YOc?n1lRLeMf1=x$8mX46q^*MEQ)e2^;CFugy{L9C z*_#Yd@atiJN4JTLfW*{Yrlqj>be)49LT{PVqsXWB@zA=>C*4FfZg;)x(TDKB=B)X4 zYmKT{rcZkoPY26e&}Q3-Y(@ptvYczCyiz8M7>lNdSFw*K6^cH#Zmf<(F!mtQ&5tgX zE{qSD?TYSnQssv?_!=&GkTC%ru$7-I3}cPlO%%IBe{!WUoQ3^ee4rJ*D=s^oO0L*I z{T2GjP+?~>jXkirJl&No8tIdiI5lLFc7s9x-BIIn7d=Y6D{pmr76i+YleVY->EdDb zWgSxcd!A9X2V8HvP==+A#j(oh1r$hA_AmYb8`HqIHKMi8PuF{lgbt}#<%uxW#IDVX z{-?*#lmb17KI&joL01J_=Jnmys~IN%QcU-Hk%8sNgf}snn&8e>KNWkJb=okStJh$$h)y4;3h$&_-oDVFWR6XZkP!U9WV4iyxq7E|%K4IP)iq%$ zK(EA6!h5j8gh{*8vVY>W(H*q9EP*3JrJ|DD_%Z_eOu1wdbylfL@!lKvtin#(RB*E- z)vuR-Dx@ZI{6j1=;{;Jx(YLa4je5#G+25Z2H7d`=Cq!D3Fcq0w>NT^=~p>QbOv6Lmz5DZ7mTXRJ*$x)Tba%o`$7UyP7U0+MPdBNgFlP7v*gg zwU>ZxC#`%tQc;_wF-Y&>ce?KRyxe;LFS_&N`|LB4EoL_WN%N+Tf_alvh~CAA{B!(q z%WZP!KrRoubZrEisP^+|F?6oIX7!8e;Wt6<+O_sWN!Xz733BtyKqkdPk2K;)Hgcxb>_*AAmB0WfGl2lzVa|MNZSdjGje0$GFw}x^bo-m1p9Bt_u(0S$`{R zt}&&AZb6C0bF)S29<+5KM+2dsJ@nUt+A=zk?P?{oZ#g@5@4aBDH*ha$y{4Ucc$&4f~+jVabanqZB3*9qxc0 z`bZispJbgK`M_&=A=Z0HzB{vA;kgXAwFMyGBN{YWRv;d^CMxnMd8)A}f|SJdl(ds>I$GHAf?Pj*EEZyWM#0*#RUNQ#^$a7|U^M$@WS1nE zRj@r<54=3#NR8sO=bip6qhm*j2F}9)G0E8`T{TLF>iLx<0fKA}b-b`S!r6x3 zJ}BT&;*wJ;%Z(x~LMdtNs;ATxMA1w>R)04V=owdHkJf1J5^r|+s^yIG){-x610ZRX z{*{B5F4j=AuJk%n=M7-Hi1mDZ?iCx1{~qm1{WRtOFDK!EC2W@_?tz-($oTdD9=a>t zVwzc=AZ$Nh9=4!davXD&Dy#RWLmO-a{YN|7|AeRsIQpZ&k$Dh$SV>O~ zvhOO=mK?kTUR})mtxHX4@eg!EnSwashKi6y$)9<%!I)>=blvvZLasErDP^1PIak>F zQK%Q{uW050s}ApNK3klsN@C6RimE2BK0e2pS(g^aVbi3G@!xS^fjJuMOO@RaQO>@i z+LBT0jjw8EIpFVJ?Xoc$72c1EuMFpcO_u~-zrM1?D4KI?U^&%cRasY}9bclLh&mwr z&Rw10w@lqvRe9X?1l8Rd6D2kuaJC1AlIi}=9BeyM87Cj*$Fw$&C+pt|#x*?LB!n*e zW=@=th1nSUh_a&E{v^^Md#dmX&n_qTgo#iL!P?W-OL{Qe7oTlafcvTDVrZ~&G5O>I z!JL@Q4_(fUT2Lo&G~q;5WgA3EefZtsy2FiaGWi*8&SPm`x0r)kGdTg$3DT24@us%fw!1KD^*lwx5h`wHZMJb!Pib$SlIn;9P zI{Q$|VD987mGRj>TtC;)@o6JKN{~?*kG!9FjZyCrqG-$803=XrXW} zs3&9(HmT1S+6&q-GLoY~oW>!XLZlJ=<9@j!2P%4<#3?8KfIu=}N6cvK7RQjM4`IPG zYB~1({hdfMgx~@7Vh2A278FWbv@pexx|Qj zj%-RXNsHEaOYz>cUS78oBlv0%&)A)$QvR6W^?X&9PQ&3*d;&C*Ll#@u1E05{a%i@xr%Q(v&@4?*}RWA{8ar|bIMzCLty6(;cVOPtMzhNoum$vR2mMtkNE)dK%fdM6J0f}!LC@1SM_U4=ozY436@N`M+6($QR z{Ml_lRyajv?%V>KnIi;G(uvi(*cT4TV@_sI3nT_KToG%SuWZpfn1_(?I@d0#?;(2@ z{EZ|heOs&R)+GKw!wacE(90D{l=vx^rXQn)$^zugk|ro)0&U+*d)p-67`+EGRE9dd zv*FOpHY>5e`wB;j{azVd0 zaN=Ab?-zW>FZDAg_LU_}LJqjtIvg2pvSCDn+(HLqXS)H0zVN&sXBTBuY#hPh?@>a# zqOaj&b1LN{*yf$-f;oYe(s(JJ{=<-=Qkz@jR>vpr zxiIh}TB+nsyCO0NO9$5ee7r6+1MDb@7(9ueEaOmu8BMB$adt#?+bTmA zN}{U_lp#%)A@SbAM{_bj7xh|1b1HBu7RpW)%c74CeVHROQkvfag&2Aj@>#FE=Mv`X zPjm45fm|cl--m}23IkVSxI z@w5JY;MWMtds##(1v067pGHlC?Eq6q$h(x_3j85O!D#yK3sSw|Z#UN68<$UuiD|CG z)x?|qe&ZFa&n-L#VSs-yt38y~0mtKiMGlAQOmOgLi_5b^7IaHOLuT+O_mH0a=BLQ5 z{qc~cI3hwe8>gh7aTqwzPINPR6TLd8=$F?VALpvBf~^D}=eCrBreuz(ubzby{p80l zLOV8?t1u+c&v%&5S_)(2)HtB`gJ5N#ZqP!gixx_L#Dh4;L!dGLNa71nU}5T~b14=5 zIlV?&eKZf-M^68*1lLpW@3kUaKgsJ6jZ(C!%gNQHNl^7fwa_1eJ8qD&Fii668}gt> zx8*S%Q811SAd*72#R;1DzwnMUxs}mk5wnv&%}l>W?@vCxzn<>}07Hj>^`Vn|Zh)L` zw9FG;;wRa}SE2fW&=iLLB+||y$7)gM`n)sSYjDaUpIbxbDu!)OKU@RBXS|{y){nhq z<{h2F9%BHV+3E>b)?!oaFX-2W--0L{e|!n`3#Y_aYbTD-f6_Pm0y)-uw8KyuDecDK zqKtU&L-W(=9yQXRfFN@M}*ednLMh-K!hS3lt%yT>1#<~`wRN~2L%REAG^Y57m0Olb+sU+UUdlnPV!8CH6KxZn6PI-pC5jm{amtqkolJRFM;}!- z5UT!0Xhy$lN7WHstUWmme6z7%Fza>eT&H5mJEN}M3Y)l0Va1dcL29`xqTQ*9T%**| zCp-!Th#T-2qF;NHCV^lP<%&V?!u*oN#B_JJcJJFLAinc~3`^rrxm<)l;La$1rPnxJ zj`$F0mA3bThl4AcqqEZ5*yoyGWMHS5c2n=q4rk}JNoJ-k9TGpvPMNqKpA~O{FFQG$d*gnmTY=vjvy{=RT3D5g&<;tOUNC_d7frFdR|;Rj5tPkxJOlSTwOWO=kYy!4Gb2w^L*J%Lyc&n0B} zAAQk5{zulV5sAHUGz7fYED15z%(0>?p&xaAq3EMVWATgP6_wAVnl33DRYjvcGKj{f zfAA0D=CScPf0gA_m61gf9ICzWb_7g)F|iwuokTv(f7;kz>A)G6J*yy{>+okx;H(Xh zpJ+T-A22lf7r5}pOA6f2!mrly{e&FKD5A^U4h1f@0V}o(E-SfJOkF|6TI_4#9wHk8 zeqA~oQe)540%N)e7O5ll%tV)9T$98D+k1km9hJMdGhx+n4*+2KB(yjpNg%G$;fAAx(2&ESYQkg2^v>%=3HL z7pVou4w{O(<5M4;>k;V!`CQ`fZ9nT0Ac7jjz4Q1W1IAFxo(px(fJjDB>R68&36vxs zzs_yhkR|!f{8+Q-l8?u#Rq5-QK-s}&rNqQ~aJPO4i+Q*4F+nN)rRhstu0IN6?;iPi zYnfkc&}V1cmL{=P))YngO@nWwB+R){%R0o3<172)uQxO|9{LT7np43jCmMO1b;5v) z9NcCk{H8ypy6>N*ypD_AR5H??J1&_ zdg2^zmwaOlRUs?qOSNm(FrVyUU|dDLd*dImF~{E*N<{bhT87*TOn_kk5WvI- zWa$s}T;@SOhrjuT184|$yzuSgPdDQIX$>PsQ|}I;l}F{fFLMlIjG=U1GrTT3^uze4 zeE#?bc>m_;gNYJc&4$E*)JY?;a5Nc&k;EQzuCvHchXmirw$YlA=QyWfp!@Yyy_ZC! zO8e2zKz>ff`6JGl^pE1A%=7y9kASW}Qu2X(TgYZOmeMe)zituX?!XL-A>>-V>kCkH zm?M|rFD_yPN)!uAw2H#9b}TqCvXByl9Ty+>rQ*XtGD^&j9lECQcZ%a~7zP`lW^0Vvrph^*W>Gvg`@X9x3w01Qz@WL-0<|7My0TTG zsT{6#oUh@`|Az$qVfM@6>!Uy~ov2E;qzTo;OwpvT#Ydk@&p!_A%rDBM`X^fj`dl7@ zxX)*7zlNWEP9?y>)z$Ef&-UrI8yRbSPqYL`Sd>k(?Bf0h%dmFRs6`Xt111l zqoqGGtNa8kQUgTof@bAE(r0(qX4Llca%N-P(VUiJV#Foe|LAPoDNVVemvbyNEsqO- zjq55VRv?#4%Jc$@pis*Z6pJ~gRYI03s2EMsu^NVxa+YSk{pus3K6`|sWVqt(p-&OC zCxVT$c9}}vlBAHAsgAqghyM!>B8$8S{vg9%N=z@VHz};JxLLy%U1=|PQcnpRAix1~v5!;;fp4p^WyDD zj|@@tlmLoXqP+)wI7U?OTe3DfZ=~hc3E9Zqz^}kP#tO4@=L~|0g}KBCi@XGMWNBtrwqVE0$p{) zI6Lb3o=UaPNf0BL8UysTm#Ga1^*6?y2@C4!(@33t`cw#|r%&T}tfx-_<50{))HP8N z8Vq`CoM7KD%XEpqJm*b0g zkeKAkL%#a?#}6NG9+Jtu0&2vv(98gYzC33mvOp!E(xdtJ8GGi{cudh)?v+F;M|Mr` z;UU^1$y#*dA;@uxufaE0pp%hFonE^7SAQk8%$FnzuNO@|ha6pX)Y3;JGzQ1)8x2CX z6d@a&6q(O1eik_YyZuLe8ekKjL+m4J*+FQ9435)`SMmT}oetKY|MkzFKYyM*|2sW7 zyEr%Jf1CJJod1=q@u3}rhjNV%qcuL1Yiv#j&FNsoE4Tk(uVZ$39fY3*v)m|N`6yUr z+l~$A;Bs@QxTA-PbD)`iNw4|>3ZpEtScisl2ca8^JZOPHPy;!C+Q#ghFL*@QZmvYnN5 zCYvTaaygS*E?MYVlCw5vyfp!oa!a*kHePc=S8M#OO8@teN9feDp*=x`kic2`|Jfj~ z|MyP@r$+zZ#AgouU#;<>Tw|m38>PQQ>HkHt$ZNqNZP4{~jy4MZ9x3=bTs6?@%Ns7E zJ||u8nch6;=@AY)r8XCMYU3JmPWtYgVq8k!iuK`r`>a;~r>yKk&@BD`H1GdEdH($3 z%;^7{_{^{WQ)Vs%8STGX`~SdLi*$d=%zdFf)ch%9i|GB7F{AYxt-q-C$Ncn2I)5Z{ zcU0rgP-pc0m3~&M|Fe4-6b*ox{{Mc#|35ex82x_}pZWEFvBt&#Fa|)S0dR8A{f3(Jj4Dmy`#>N;h#z3Vp@Gr$JH;Dfrc%?;lL4nVW zk+6S8LIKbgu^0-S81uoH4@L8#oYx-7mMD?iJ8D)`Qo0uw#)|ZRiWmv0U5x$z+f4ue z$=P}TA|L;AX7vBfeCE*q<25$=zR~wf^!*PJdjNfoZaC!V7p2eqHyMgqs@G5t<+m~= z$zh}i=^E~~+Ya#_F}A`U*$VNZH?R#76|&gcqUg^MG0q<1Z>~B@Gwk%Fz3}i*zNfx^ z?R;I2IZ(K6tQ6!_;nmE7?wXYHsxf5l;HIj%kQ^bif(c~kLH>L7SLYh}4)(`iKB!V= zQI4Z(y*+xzWx7R0*J?2?q0R9fx@@wNkE6irf_Ue2_9AhsP}3`R>w0eUYaCt5%o})b z^-fZs4Q=IkQESJtKA~B7)q2txRL=XhNYBDN&mlyMbNHnEw}NjZaf?K9waF@9vd}X^ zwcSE>&fRPWq0xV~LjSRY&?rCqvrmok(?;c|nYc8XPsJo}R`U@O)u=uDAx`Vo*~K0< zexGgCdz$-w)H<4-+U}id6rZhEd=?j&M)Rqd<*oDis5LgKPx}SxTsztLduH$6CDoqQ z`g@GhvkQCFC_U{{dW^59MHjKFBvg zIJpA#vAvtSly*<(txKR&QjJDVxkZr_s#*27=1-bdEUA5%WmWWme9jV^@2c|w1unMX z4dR`zy=n^<6aLLp| zKHsQ?r6XlLvi6x$WmK%j8C7O1Co2~a=Q1mrE-8}h+*T>Pl%RbBkw^RzIlF<~BVWys z**s?Rv~%;w2S%a$DBFicFE<;=Y@oJop#LVr&2OAP$k1$<9oR6t;4I(4&r<&XTCacl znUVkF{Ni~&|HtY1smcGnk!uuoTZVwrvHt{*~dDBAx<#+d>k4K0_u6ODq zj^uG2CGNEgeZ7GYZ<5u_(ZNm}9c-;<-}v))-JjpUiysFht+nR9zK-Z~KqyD_D9Q(T z^amNfPnLWoW%(CK>o;HN`|8=+u3GgA=#S7Oo*k37(L|DS^GFSgl;moB zG!AK&T(RYhvD&(1JI)G)T3sz){8bmD-tpxxDbU<<23T=Lk{w_FVwH||E&npvkc#0} z=~bG=F2A(#nr?Z9l6G-HY^F57VX?rSh{s6SRmP%)S17}#nxw^9wLrqw-HWVF3eheO zUO2JIE(@`xaxArL{dY-%)idY*h_A&H+O&&z7tc+w%ff59L@Dj!fYwT{`Pp-S#Mcs8 zDcZ%o3tlsR9O1R~*$&#Z_PZmwjMlZ>)$(@nZM-6ncp<8U%{Foo>R_RpOI59Et{ScS za!`YPxuuta7K-#-*KSx-kn(!Nif3+??Yqok7JcoDzMYpa%)+mI;kV=Rz)mdmDnscP ziT@T$4Fk*&y+ia6-a+iD;MHk)TzGx__v!QJC(m>B-}`4LrvBqbK2`DGv)0&TgW1oJ zZqe*jfNvyU93dYLUF6gTe5(eEkzQ$hv~^+Nvsm7Qhwof?IQBWBchJ2Ar^+`H#f#5k z1^H}A967ujTjUN=i(?NR*u+8B9iqd_q_6k{EDBS`zSv$24JN;x_MP8_wSRI3o`KVT z-e=jktJA)dRe9EWU$SmfJ-}=d2p*0HaZns3bFt-wRQ8)(WwJ@=bLC!1T6Dht zHkkb0tvJ(^JH(RjE18PwmuQN}R&l;j9-8ihM`7rky1OzS%qzp|;&!7e8+LEX?mgfx zBEyHFeT#TQnQgg^8Cmsf|TZ!=a=7$zs<<-wzIx=RyhEN*|6ozdEtL4 z+YE3*0CUNG=8w6&4PDSkA9;8XEK}xIKRK;-tAG7!F_$E}tbv-*&>~fz!?_77-9Brg zt+@Z{iR-!Osx!#a*ahF!tg<}VY4IW#y<}zUnh&HZ23ECaiT|FQx0R zK71Pj+RhI_W8poT>BTOzjY{(Lfccr~N#w)iUD00HTyo_M|us zYf+gZmt!Muqj>P@ug>|=QRk~rKN2P`#;X8u{AAn4U+)fuuDezGmC>l0I717|eyJZ~ z=Dnab8d#eV_pnCCC{-`=5Zqot+XR3M@LLYP+26ivsct$q?CuPqwUDd-+!nZ%gm#3H z(CM`tdUaaNk-7omIb0zvwvUYyBqx~KCVQnDZdZh_BE|%T81cL2Lm0uAy;v>pf&-Z_ zxPnjfQ6Ig6p(8W>8kJ!Oc~*wH#e7mJ$OQMCdo5$%3gmCV%SHejW?pqzsX7*~59(+n zjlWbVeUXB6A{3VT!IuB!Ko+|WMIERMbps5TqpW%R5+2q9KC z29@DoM!$&li;2-M=GQL@lLm}_v7~-cGNpSqD<%3xVN&)$znHXLznB>PVqf%&iP0}+ z=ocqOznEXYC{7wM`o(hkMcI_@*{l@xi{hm0zJ77icKzbS=okBob*gBi84B?Y4?_qph#> zhSpw*_55V)npkh=?UyRCKKm++5$kimc1y+jp4^35nOL8BF=nsidSNzpO|ZB0%0ZQ6 zZ*c#>i1xW(yQQLiUoIf5PPQ8dd-W}Yos;Z4)TJR*Ci5L5@bO&;8mh73&w1 zMG9P*Pwi*<`d<^wI2lvuRTcoNum9CQe|~zBum2^!GWEYU@M%{6E4Rj`0GKHNHh%%I zpJFE3@(N%%W;Z3k_M-$?ZrSBxl+*xA9<%-oq0h044*EwgUi9)-QV+PO2Vxs!yKJVV z6G3#rOVF=SyW?rO1W(1pJk9Cjk9g@QBY<{6A1@_n)e3pU&k4Jiln!PtF0kNbaih$) zkJeO(xzVmHrD!vbh~;Ztg$irjL|p0(Rbq#w&nJoU~XgN?T3*k)>^mhqG1bxbWr+w2Q$g=azDSie)l7# zg#HjZW5j<5sSmy++kXUIhPpAh9|Gti$}d5AyGQ}ZO+?KUX0(-8*$JIBgOzQQ`TWN` zt65Z>v_8vs|6C(%&}HSoW8KR^%%(WtQr>)=An@w?9Z($r%)l_DzCaL82=GFCqB50$ zpm2Xsb+ zc0lLE>m+k(hl~?XPH9$Su2$i15enX7fg^7TO&f`?t&#lRlB4NF)?lYf56>G|F7d~g zAiAd~Cj{LgiutqwW$~jO(6K4z*oN*5A^~6%B1A9GF%0&Yg z1}=6SE-n;YTpab6M2pkDSJTrxXfZh7YPh)AG+e~DQVd=gyfAoS@ZxdJ3r9-%)1~p^ zWKz@7TyQbjX0#YMOQOX+l#AFDEKVi{D-2c`tT0&d_*ikOuwt+@RzyZhc}MeL#pzaK z#p#Bz;=*7>3$S7}aqx?t|7)jDV|jM$R)&AiivK=4>!0T1zt2yfoAduoeCp2stJc^= z0_qdS0)%$lc2G{wiiRC;z*A@?W*;dJF*2qEh9BTM7Ooz!o275f(wnQK z^0NZ{U+j%TwG{-!7tMef@xLc$7rFbNr-S~Qex-7fgy|E~IyTdcI^_0#zK%XsLQf{;rVZa9FXMnvqO+FC*5s_m96U;2pO z6MAb=WFvfstl^ZSh(jNl?Mg>x7;)DbkyGe15A!YL#O%-vEAA>QE``v8v}!Xe)#p%we|ClZ$BP%m4oR(P*qOQI`>i!*g^RC>Ix=i zUz&ZH+n0Githz^^@(l|;OUnpqn5WImQ?t6@cpF6IEN(PXti6o5tH%Y4`u~|dK~Cr* zS{?S2d}ihUxHx&9&;RlK#N>b6$fwT#pIc+&@83^^TfPZ_5#9r_#)$`Q?Nu0P1|eX`bx^Pw~*r84b# zMRvFiDV*i|zQx*QJwa>F1lO3=mdUr6=W6XW&>a(s`NT84ZFqZayxoZ5R>|Q*>^n0B zb`~@Y%NmxwoXtUtap}^^#Tt0kuRL316)6seZw=o*f>y9qN`Yb69bwp&>ICOo zs1uwUR^1U+U58Fk&avB|6BxeT6~0}cPEgLhkE9bAhTRp0oku5lgdl^3&;L2(=+Hx) zVtYpUj~VCxCxi1r;r##fxvBrLkx%XUzrM!i^nX83|EtdT1#kSKSOe(uqyOQ*$xzHt z{eksii9FE`ytgl8Kp&|I7})860ez|UaDj)pay9B(;%Cgb%+t^VnzOMe zTb51PnCWr8jEzM`0GJ6o0#S^7TL2MK>j^o4K0q$IlRzp9rDGqsm~o&WQH}?E6tIRF zaPcj2r@)3G6TM(Apii*W5COqF;#c%JAPj-{&2UIUF8M+wb_8Vj2eLT>HlY;R+?@(b zK@7xom(CrTrCNR^zlpvObRnG^wLFr|C#__HqV5VwB@YtwOmuI_?yabMOLP)lB8USL zO5y11*G`Wj7P|bZBZrV!Yip}H+11RBV|}hxLCpF`pAP?}OtBgLT1c;1E~&OZdw+RK z%}Q$KPuRaH6Fw(3+ibj7kh01!{v|N{Uy$3AML}F^Y*4&CD1I-NZ5Lr&61EZ?FHF%w zKrVaP86=;Fn(c&gO~C88CZrD)3z4xC6Y<@d zm7Ev{)8|W<%OK(F}ZZ1^P`O zEWbf_0mn*cGGJ!FOm53xFq5XpVn`M@H!${?Fc!g+05VUlM>RN!hm^r74Kjs78-X9@ zWq73l_GCz^v=C4Jp9ep0IrI8K?bmQ2(Vk49A?4zFBkC!iC?16jQUT{pP%LHKb@VOoBS^u`P9UJ zC2MR#zf9`Fpw)@Zxh3{!!caRD(m4}V6LnIQ8!lL zqOnDh49=K&9l%Ar(~A%MtN`-KRp&Sk8eIEHPYJz=cfMmMm{zEA8n0|2grDU zG!KyCj8{Rh-m8DIV=hPiK6a^yPRpft5F`&VMJm`ruamZ^jNfU8{Z86C>z=3iLOZUB z^O{07`I(TQaX?3|(Qb!v;2+ zY%qwL7x`mA99z7>&G45N`XOclGaw%V(Noz-4-bB}$Q_~<$6gdR9}ve%NR?p>A+sV1 zEk)Xc`tPbpU=Pf^DUgcNtH0!fd0a^X@WIO~P#GDY+gxppN+^<{#Uk88>dP;;Arv9I z!CP!UmJ{-baf8>yc~AHU1f6MPV2K+TP}vBmtOipikbxmWzQ7C!eBvKcx#$WvMuY;F z_+w#wO`s13=fJ~$$PrsQf-nmjKH*g3og84y9;8Posv2S;fRKgW8eD|Q)XPj>J&?o# z9%vhSJz;;1UJXEkr4fP#6Ufl)dvs0Eu?J*lSATV`kxXgz<&%HwlY9TmTZmocbabh< z9=~bjBxv4KaxWAJVbOH~C&BIGt;L|XAznyH*2j{tg8cJC`p`F5o&P4o)KM(g>q)wD zFRQ~dH3(?2>Z6fh)Si7)d)}bHB~uUiJo0&B;?H;YudWB97;Qu`N|ykPlQoc9!b0kj zlXjPAO%6d!@|0ZHp7KXy-@!KIhzUi613p3eP=pRMl2{0aq54%_%RoM$#FgJe`*xLj zRXXx?9;z<{mQP#!Ja9sptRNta;h~GtF$Z$G3gLurGpF^qIbx#(wD481Ozh&{J!dw&5J>qJ6ZfpP{-208Z^IX|Gl#WrM%;%2_RLCsC0=E`6)206Ko$dQnU zJ_pF=$N_%n4G{%o1ZcG48*(?KBaL6&2o&ceraHlmwp|Mt86ew=dssaz0`Ri#glkF-0n)#KzHEQOGo= zDo5_-2zn@~R{EMW?r7=&ZXQU%uUT0Ee>F7!2As z43gVs6uLKPLF2Nf1A|5zK_hJi7Q-MGb->Wp+n3rFz}AX}9e_}C1ZzMv%MqWX4*Q-+-wASl4f)(4TI|Ar zXp2o{W!01-9|`%IfKR>DCtSUWBs^3Be9Z<~YwvpSb6i>(98h$JquZ|;3#i zNdE-nz(tVpb)8zrgPL(@osh1bS<7b|DL*BcoxC`&wY&mb3eRm-T&svN z3hTZptgqu7C;y7Fw=~jK^R0}Ux)C)sAkK{33fKo3g>{xL3y2Rg8GR~}{gfte$R_+L zM7n?p3F#^|+GI5M8LrD^Zb-?A$5$LxMzh^eZiaYRrFY_Ir5aid3fA7G&+q9J+a_&- zwh}T7s_h%9CHl>K!f#?#a!xml-qhgRW|G#W0CXo_ox;>LaTp+@H8^VGVugvtvC`yp z$qRkLS&o}ZoiItc;Ov-cLoOcC;F~p)HM@T5@T_zrr=Zqvr@@vZ#z5QNLfbExEedUV zQv+=~4Q)x)`3aKg>d-VrmoQNE;W-hHtq1pXe|96>(;H-AZ_|GBM&P1vrPg}`bHo^M z**9?cZ!&yO_-oW1h_Z!%;tw*E^?NcGt2w+CD3RtgO`3JiPEY#% z4)_8lp$C2ZAM_?F{E~LS39zH(U!}kA4>I%#w{wi8!H1Xw2Ro{UNAwuLKH%6xRTh&> z@!bW7>(7N5PjtcIC;kwj7vqp}nRa~jwi)!DtaYZn7up=(t?HcuD7)Y#keo6X^5jH^ z!JPfX9F?%QG1{od4bHR;XQJy+I$D4a$Rl?$m52J{A7lvNh$9NHe}@>yO5g%EA))I4 z+eMJ7`&W`DRtz^IowIU4Q!@IZc_N5b53WkK)>$FRUTCZ3p%Y*v|Gr87p8~j8a*dLP z1`RfX20vZDS_};)gyR1YpF=kw4h%VAwpdV5pIIY1$auWO8j}FdPN*R!B|rM=kynS+ zN(8BxsQl5+`YyxdFVfB!PJnZk;?0i4Z?7+{P znw?Ap>f=n(Zv$D<>#8QFS_!zdRlO@G#VC4vsObFyKhJR~Tx(zyyIoN1)V=EHEE00* zeBHb%=Lw!Fy%H|S$E<3Ewb3_^j<#1#6}C0OZ4Hpco4CaQrjcNPYd3&P{rK3G?#-v{ zL!oa^5Isz5qaViw!kJb}|5(BaNE4bGmQK0iM>IXON5TmR&ue|G*i(BH%agp5P_xBk-a$`jX;kJz8b&z?m2 zrH}(glz8B2*&07pWfs*F6V3y6d3-#^d=d_2A$_elJPAZ^gi2n~s>UCUF@N%gr~_na z9Rlnh1}?PGH{hKQ2z7)~LgGKlgNG4z5g1|0IG_kRU_@M(+{=q5*uTAeQa==%cwXpZ zK0Tfw*E_T)kaBj+DMH5{#Qt&g#cf%6`{N0;Z=XCn28WR(lfeB^V)Fk%lwsn7%d14e zK6(uUIK(dI7%{1cM-LtS6DQBdV!F`NuXl)2>>y3cl~+0$^6n6QO}qe71UfIT#3nrI zAND&X!|4{D|Nh0%i=!8ZFUp@T1r3`b$WfKb=OP_EKE&Js^eM{gHkVfb`?ec82y{wk z`cQr=FE$-|s-4t^J5sB70s|Zk`4Xg(>c}P(5hhpsZt&z5`_3h}PI?vteiEmm6Pkyr z&j0WL0*ZY;0-bN!;kT><4(i8tGzwie8cbI`liwO0@C9%h`Zl;Yl|N$dCLE3MXV5w9 zD2PlnJ2`?_##X&Dc{-YatO5mtpm8rz2M-T3R^Ujzpd%4K0{pXC7MGjoF*8i5=)q#o zP0YHiOutC)qcZlQlpzB3nhT`_4OwZ^9-Qq&k!-LY;PHXJe-2c z5M|MnO&~=~LVzt79B|@Fj0Qe&kUSm_=}P0j>>wn0Uej#$crqH2T34IN7J9P{=&#*1#F!=FWQX8D3|jB z+cRGMzHIKG#mxJL=Dm1NSx2C@-sj2_vE8nv3W}^D1KbRz??V zkuK)R#hX(dOC#{pu7^{;VWmtBeKEbPitL7}nQGo#J9~WPE5%GT_Kv(2P>ZvVt_W<~ zn3qw)wvCpy&woJ~%cD?R^|2BCD3;vXU0YVSdA>jVDk-D^hOXNTiF1t%25%UH-_Mo$8#53x?2l^>8mTlt8vh z%$6NO$zP+CzcjQ?CqOL0xT--AIFaOL)BTW@9*$bt9HFkRW`I+&EnAIEIu!<*_8FRf{A?rS%!N#G69Y^J zm{t!=D9c?nxHN$b#81nFOfL3tjf-SX~bu@PfPfLVYjM!t;za?u@(!)VeTYLD-*Z&L&rI!Hs0?I5)55woHmkYkBA z5QkZuaOgfHPkwXN>GXiAM7zcltW2drS$Qy7(&{6+8oIL7Y|HRpf&*}J4vxW~-!HG( zTaZ+3MC&plj4`=WP;HXYWIoH~e_@&@z ze~H%EWPdT)Ukce@ZWM2Tb>@DN9q04lU0zk-Oc12uYl>R1z4>FJt!1*u?0)u`XsMro zMRUho3;tys`TYM7H}ue3>^ zjp_WHjTEbgL5ZUxzZOd2knWo>sCB!$h+C?E8D`5Ycn7g7_Nyf?i6=hsL~x?d0ihhx zqp11OA7uDGk+IE^zxF_ckX8WsOvY^8rEUSKpof0{v;{ ztA3EL4PtG6p_LG)1k8i{&qAN&VmX$uAuI__?5e|Qh_cibPJt@HEA=t8#>F&?9?jik zQ_Qgq-EDF?X;8X|=Vz-NP9?NFV(fxZ@OD7K`vEfK2&qhw&~pHTULcmCCr|TtTlGVq z17vemvQH>cAPnTYun$Hgq`-kwb_8Cl{QPrkiI{;z!)lkdc%Vs_E!P}%-Eq6M$F9^K z2j{KR9`$hs6gk-DmWzD^>A^NK$8Q&t-+SN;JOd}kh-b7&mGO@nmC&e!yPy&pU#aqy($EJa z9%)NRo<=k=gIB1ahSX?ykNR{KzE^=9Kq z)t}ijwfRR%31y3E%c`wWk~i!(O~oNl@^H2SyeU<70vS+QMoY#tfh--!I(1H@^W}^K z`w6@gDkf$uj5KW!P;!SECVt^MYAG7UX;meC5AxruzdCOrc9HYt8u^Zpt-YC9RqM)p zlyAJ2raP@9#<+5J3X{yu$`*c>y#G069C?Q@zzoqlM9VLN*5Cg;xj5_R?|(i&HTC~D z^J#GZGq=X(BB;3tTD%AJU_$3e8+KSZ%B*&or6BDFw_ zr8SX-T6$ybiNW3>3dhKi6*^_KW&jxngTA*)v`(i42Ds{ow|WZ8=+B__df-m1L?+-V zu5_6u{f!4osHM%n6`>opQI!0*fO&(yOa}ZCoXrIpC7axsUIruSGeTJNWMvk; zcWM(4`(r?(g;UXX{;VYa-zGk%#C1hK(1+vPMS%MF|Ni;e;4~Zm ze>xbP^-cW$CO(bh|BGvEE&`Z~0NO=>*9n9CqWD7__(6HTg+i0q1JnUW-L1b%P}~vb zQo-I_Dkv^~$?XDtZ&&jA0^yVSuA2KkOPlVtwOt{XmNmJH-Ole5E%x!49@3PhBaYsT zzqB$NPjSa&bMutAR|n_be8F^O+hLnMnx)L#Vq>qCo^|>*i$6YjY*qYGJ}a^RVol=n zwDbb_Ec>tDKg-|$K0CWG_TMHxjqSg5jg8&6pLU;Ub}ct~thvsx2<5|I>+QIdvyJ_> zH}+e)=p}Yr)6{CBd2`z<4wo!qGb=gCYt%I7+<9$|U|G9{lXaAaI|DB&azbNGY>JQBM-$p)j*?%tL#`xP$ z9T5uv~%6sb%mgLB|vKjerVJzb%JDaPOvPBmuG zY)ru_05i#p@{y9X)}n_vm9&L_P;iT!R*Ce#+!K2166i4C!6_IbKoPSi z$O#1>tXiCh1=c8$=-NZM%gvfl50Gtfh~wN+b>jRMO|Lqll{Upkm>g^G>GaTOglvA* zc~5R4>N{WKszG?{OI7^f7EOEd=0q1PK9s*YAIT5sjZm$S^TRZ%NbxGbiEFAt zQ0If9EH)C0%H06qAi64WRM-#{~e%2*hg1 zEPtsw*(=_;&*hoKzLs4q^`oxX?oJ39Wf>`NeHRGE48%=}*ZviW>|hyRYsRx$^t^uk zZ6LECuOIndf|EM?xUd!53QJYQ7+ZNC0m?=E+C}3xp-Qj)6CO_*8_fBJ2AnnuPO()f ztlFRP2-#EHMc~If(Mc5RLeyU#Py103E8t&0-^Tq1t}6uFr3ETTyOi^~OEM#HegI4t$J6^|!Q+<#dt zGw=}qaMkG_y?D|23SM4;LFHj#$u4SXHkoot45;iYs1WCdL&`q|KSPdudpaMol(jbS zvT>)Z5UorsL>Lqx#8oRJ1nia~2yvpZM?;+rBQdQSZ2~n!!z@;m)|YwlqL-#2OYnm@ z);*!Oh!)@@Q}?Cvii3-I?t()PM8Shr0Quyqa~%B|SB#OiQexdCeoEk?u5!s1|5)NS zjR}yj=tB>!aGc;Jxaco)L8D4W7|7XQ$XUALT74e|el`X_C1uG*geH=@8Ja}Z%cO^` z`m`|CSJwqXbCqL@2`ni&H>)gLEUooQV^NBexCj~NZj3zn8l8=WuBmisM6VIOM)Vrd zyM*Y~t2wS-@^059Wg~jmEP7R0k>x~hO&><|ZcOy5RoDs9tJ12^DSBCv{$)gOlmvbi zT5rw%Fj}t>y+-sJ(Q8ESEYZ7=*4tR}8m)J)L~p?w`ha|N58Zj{g3s*62;hw$I5w2D z6Ue{_Vi!3on8Qc+@`6W+^?uN#lH3ja z6e(6%@&%R}lsr}hFRSmo&)iCF4VG*|n9=|D6od-O$zmW>-;DvGwt-NIGQD*mRIn13 z1fe3$Ge9UZ5taa)`fbx;!Z^Gy%oA2Ia^G(+L38*dNS>*q@O^>9D z<_1Wn$!?n=rSf8VTcp_S{02$V$D^7h38y}?agy$Yl^2-Fu=_UKBc;sRVxy#%o{d{3 z=Cg^TUY1ETo9)dC2C9s=H77;xf`fIvVT`#I5#0mn>!e_^ zD3VITK{~P4FvM<}Iry8ul+<*5_C-LBj}h(O@g^}8gbOocZW z%LDvDh6xZJ7ycS2Hp&(dK%WCdG4gF;T+LTPKn13)v>=WmtuExQays&~M5D_frB+Gc zAW2I_5#w>RH${Jjh;e3d!lA32p)kPeO`V=i9CYlUQm^NqEyVRez9SFEv)W znhZJCXK7w~9+QTYh?k&$)USkIUGlUQcXKot(A^!-ZK)4?0spTosg)cCDW6cS2e{t< zd(uBYxybu}FZyT3|GSA#h5xr?jg1G`cz~nbF1X{zh&etWn^0#9BYR3%*f@fB)Dc{= z*19WLS+Ny0#Dr!+LJkh>AnXCJQ0YnLc(sEt$xN$h(3$skdZO`F2Rkn6q&h^b-%`0z zE)zxgQYDWKJ<<6k3%$+hc$NFNT$gQ`+Bbu`Rp@>d3^f3U0N#x=ER|&@7JBJlBVLoX z+8kUadGzWp$!A|Mxrdm4VH`m>Hsczyo_Jfxv5T49c$Z=v=Y|Vg&xHly`Y#3G@RCUP80GjQz29PIpRGJd4x`_Uy%3e4q_J$U1SZX z9I<+45+6b{zh*nncK<^-8X?M>_6)#xHJ!bN_5?Yr>FXEdy``uz$@QJxxNz9CC*3tz zRxqLO(|D*bN48 zFuS!du{icn6D`(3ci0ws?7^{v?vA_v-+|}ZL-+P5IQs63f9Ij2@4nEXbM)O8CjPk- z|9qDJI^N;YcV9gDBNTt^)-#@^UUgYHOv8dZ$b$C>k0%^CZQ;fIcnwpwnJG0bIb-vy z<-BzbEw8qR2-0E>at3GwwH69hYSJZl>R<^iAlq?S^7vESDJm+h6|U6#C4= zd;<)adzIlp!-2Y7awrKlA^_!!4EGuC%Lt{lvtRa2hX2~ke{Bo{S~~t`#@)}q&5HjS zoIF20ImyNUoSmJYoA{qie451nn8+U!`BM`4v#lw8vKVbbe)b^bN6!7+_deef2ff8E zQg=sh#G<~%u8aNgr(pJNn7Y=HZm@$lW2ywea<^kjd!6rwOu7_+_{H?pN>NX)tn`3K zLSNur*~9X00l_}k8bDrruyB(`<_03ZJ58TWWZBWlsFS~?`YnL*u{e-Y1-NL$rTkX_ zDEJA4!BQ7e#q7#|M#&t>cSYTYfo>|P3pUzpVg>?=9BlI_v32MJVM>uvG-9ERM~S(8 z^qGT`GcX}x-Hn_4DReq6OHN1iF?4Ul_%sj(SErY*lKAy2I4*R?@PBHK-s}2$1_u2; z_#O<-ld(eI(Z(H3iREG+L3*${GJd<5{N4j6XU{JB&o261@O_f#J3oZ{JB5xUSZ`ih zudcLLWHL9!S6AvpU3wK~(w3VzGsxM-TRBx?cL$8|jU<~g3V~4wc1j@tBglB0w1Qny z3aSu!%}RmM3ARBe*q$7eMk(05Qt-<42;;wo{MV6Wl6lpFe0!r97{y=-#bDJM0F>-W zLQY0ZW6yFOKv3W!#sHr{ADn+%GOjkme3j|{SQwiZ9SO>w_FIz#V4~&JvBJ{s5om3+Iz^DV;pbi+{z&`5(n%_u%xr7s5era?9qZ2Hn6Rgn-KhpTK;c0itz}kuMTYS(h zzid6imJno2ZUeCfVhzL^h^;&5-WiD9;xq1jfLLS3?jOd!R~dFbsQgYp(k2xS%i9@D z-8iNeOBxmd5BnU^5wy`v%$2}`d|B@UQRI|CWWNZJAj%CERS*P`YuAO01Ab3{hy9SF zCF>q0m>Fe+ir$H6S>3~kH#O2R0~Cp=7IW-Df0`R~7B;K%tcmwkC%XFj+llvk7aZ4g z5J3AD@!5S;7x^pSd@V($QbLT;X8`H;W0di2ZbZ@>8c5nxNQ(0XZWNQ^mo5%Uxna*j zr6eoj!myO|uu@#ow_v-0DMyJxrhP`HpYV9{KJ=c zy(X@UY)P*3o&oVg%@4Xc z_RwdGj7ALcxN2CrSe4F=sxU7HQG9nWUyriX1$bF@>YORuy2O~j2Zw7J^!JU9$_p|e zb@#OE7I+yU1{L=n72gwnGxhBWB|c_KJU2zgd{cvt+m4QcF_^w&HGsG#81xBeB}`L* z!s1Y?;LyXqMeY!l@?S>+(cEGj@~{pDG7(ccwrD+h@Uyg4W`OGz7+loiNPfq(8cRxu zF?h1ic=F-ZyEkvpm_nzifu*-F=&^D15YrDJrEpqusGU2fe%c&mun(M=4TShbzzL;y zp7=;?b!n72nB6kzGzuwVN1BVD$+ATb#dnCQ?pQb)VH*p&H~}O48DPdj1YpRC2W3E~ zJ6#P%QHUKU%aIy1$q9QUCk(TM$P!PN=g6AQ%%bSe5L4uo#__JU+B7%>c8*cE@|>g!67}7mBH?+fkT6Gs2{kqdh}rF1CD!Hu!L*UV z!HwhKD-K))8J7SR;U4yNcx;A!>Imd0LWl;C>LcVZ069m#7>P`E5fH}k(47J(F%qzQ zxT5S=65BAl9c36zzk#EUaLQ(&2IFE>QPgh&RWwkULrSMx%vdUWE~npO7rluj0HHr) z5A){tEprWxynd96mdQ7SG8OXE+(5BIN|DdqtX!|ZCAqr$kmi^}W z7<4h{V$j8)i$Rwaqst3}F2>2U=kW6S`qNK%Jo&`nxS@whZ(#ss^A|8)Uw;CjT(0IB zT!skw01n9=0x$aC3MrS~?;D)Q*ZX>l1CGYZtGU!+2jK*dC!xk=8te!L1j$5Xp@fB= z+@vJr2Oe=y7aYL5ar_lHu!FD*4$`}Qb`ZuV@zvM)dO&pb%~c0GE{ZPqb-NMhXh>2# zsn>Q(TY)1-gN7K`9}$BV`-2uYs*Wk-A(axX8DivH8PwQ3YN#~;E`bgR5Ct}&3j)Z= zs^KGp9v>%|0S}KST+oMw!mlF7l5tVQNJwqO089B0`VO$64{R7R1TY6fWD`$j`1UDc z0r45WLu7MaWUfIp8ivyDoSMPO1Pdzq#8AawT*a$J;(*-7m-^bovYYgV1X@M(nM4u3>JK^0o%| zHjjI1C4eWE0t8|$KZ^sVD56GA&$ z6yIy;Rrx>Hc+ECyMw;DLo4-xcf8gq{3=`4$M(Mw2aZ_|HRY0v`A*|M=#o+@gvk`>; z(k)PE8uD$kNMfd@4+tBKHNV2t{>dJW73xJqILtG9bNf##MgTWt2uK5Hmq|K4W?A7`lG^TlM$~eN%_9XGi2ThVSb%=+7mBLFV-L3kkiN zt%++==CM;OFcfmfer!B*-Z1FyjhE~8M`5^eQ$=2c-mC^*8p2^Lf5108<;(a-%_He- zZbW>OuKgmZ7?ef6)3YqDIvTQ)_v_-Gr=xoKR_2TmsC5>ov}y58xlg9%M3kXU zn7}C=IKS%(er#uJ+(fUElWoG$3k2l4!I@f;iL-8&^A_|D?<)3G#xTyxqzEul?AHY1 zg7v~T##x{pawx9Fkx9;MNZ%!5NgR|6G^B{EmK~NS*_;wf7z8;n6~^6kWQfHHmTsrU z?@#}FQt{s48?&fG{T8W;ncwRrZ1DBU@sNJ_e;AF&u-JB4V#ld?jJ7xuS6sydB7Pwv zo)Y7FL`aDs7OHZDLT(%hm9zH7o%x%W4-9kjdw_3mE&%ZP=mD?`YCJ8SomG&Rm!Ac! z2ypRu_&)!CPz{bBRO1unO>(3w0GSNmy`uI`g%Q0|`^$8|qmMo|b~H_4U_V}tH(CZk ziqMnj@s%C7VDJ%9;`dZ;7m+eWBgGs6iT~GN-(U83h^`0Ve1rlJqe6j+TSyNw%V4TQ zf8sNLWCuM*hTyQsugyt{Cz1N_Q)nIwPhO!KHwy*Ak;cJK;n!zmt!UG(+Cr`?tWuH2+U7U_5aUq9T21OtM>j|)pZ zVG|MDppsu7=0_5PB=Q$H#HHFSNYyYvUW+O$0*oTJjqD^xn`D6sh=QwPHGug^8eG_BZxXR14K{n42g~V=g z@&dKUROPdTF?&wagmo2>InDK`b8XAA39YhW)ljo9dJ}-yr!M@7WAYm(E9+U|#}zq$ zSnT<+MefS$0ZZ68m$RCF^-mHD4imO^|AqU9s9M(j67$oNJtt3opc$Ww(40S|I^vsV zD%2g(V(o+I#oET(IjZ6Y`}!0V{dAb}{r!WWdCIb1J|T_|e|SqQXhT!W)Z_Ixe%cm8 z%sYt=!=9DV|I4rF*)yJ5a>qdm)IJ>#36Y7+K_K0giNz;vLIzG(`E#N_ONeSAA|f6u z@Ba|@vl?7j+y-8GC`hG?pxM2M+~*9C60;rIO@WFW-|@tmil@rJ3CVDZU2n`R!IPz# ze5K{=WE&-p=3!cYd-Uo9mRwHRSR7i@k}BMzwL6*gQkvCsBhCF^aW?Ojg|+Ch9V~O) znjLrpAAwm~vsw(CHq}_WRj)-9s`|)a`5J{^3cucLfpnAOHD6Nsx$Y z=_%~0vHQYbLp!2deZb!DD67f{yBy+RAXsVtxo(QC?~%*gbxM7LHFMp2z7cfi*r65t zgViRO6-YA(a2aoX7ybIbIIB=Tm_uH-_8D9+1JBunT9(!P(g{ z-=VMYiN-1!vs22ILJXO1iJy+5@gvIx{{)cX@4g55djbGjy?#G8tbkmBn_O(b&!rIB zHz0wH9TND2kdPZnlHy|3qB!w)fctsjp?|UNt%E^!(F-dp$>PgZj7zGQ+hz>5v}+8P z)7m^sXhO3-ZQ6B59i-n#6hM+HA6|KPxp`;Xu!Xw4Syop^D)-@Uo0yEN05T^AAvdvV zi3l1j2x{yYbU%Mx_$-x!9^D^01W6e{(vHbJtOA)bp$FxgSviiZ=~}KYhTpNVk%lF0 z$=@cm=|CM0*masZNZQAhGkz(9O&M}Jd4-&XKf^=X#z*ShjxyUV%T7GJye4(*yFHcrK!J?p^7q3>>n10v94iIcp@b$vR2Ejq@_2m&#S(dZ)5y?SEp$+9-Q9hB4)E=Zv6w6DKAF^mu=f|QHQUH!el7OVZ0oN>*dls`LtV_; z@!D;mH(DWxWB7?kp>0{8PD|#BjunVQj)=S3$Ts+hKKO~a&XXOjh;8P1DA24i73p1c zqzu=%=?ti`TSSwGW5)#6K`-{F>k3|V0g(|GA98$7^*2$LY1B5awbmaho|(p&3M%}i zeBgm-QZ2^-E~qKtp5>Ne`)As#i30op&J*q(^AS1d9SJsd_&g5toqY#E#`Fp|qq46yzwSySC?0rO%ls0+rB* z@^(-lzhuX3FllS3k3JY91`hGmF4gTjl;w;*zaV0>eGUjU9H+NVe55s{%5#{*;#&jr z4i$&eiuF5~C9@%o?|Do)fJ%a?K^Y+QQLSzfp|}@^4aJ(->>HFg=M_oNLmqOyjZXyB zH2g}`PIK9wDa4|Jm-HKiMN}{kR1cZqeJ99;twFx-j^Laho|7O2W;xxHLG@rZYH9>~ zH&Hj1Y;BC{!?lg!60k^BzMlo+rsE%oQsg4D8&9~I)>inf3+IVbmPD=zo+81OaE$bs zbX?#9V?iUC;#o(_vH44Qh|=Ttf2R9RqcHCC5J$)A6Ep-wi*F8Atts8h>a^TIHYUoL zFpW}hqXl2{7Eb}#|~!Be(7i8i75iUjeGJq*oa0Ye9YzNyZ((KiZ^0~44f ziFvh!>FZx-W7P63x+98z8>+_iSAB7XM^~`KSSAZ2EwIDW>qUd6@U#9<5ddI6X(s40 z92j2`n4j7I0{_YmsR1SajE{o%K40h&se;55){bSm+TGnHP-2F|LPTng*Nq~PjWJTS zc>aHN#2w<)^*?pQ%%0OixW*a}eyZI4XwxaMSgokT=&(Q74m(Wq;SMe8%syJJ8xiS8 z+csnuH&#Y$UrE|;oQ};S(lzMQK6S1hyB%*ql8CjVYU)&e#F*nvY)&hRtZ_Cl4m(r> zFUd8}G>^Znq<-OE+I$C06xcJAgUZc)ZV%)oM6hma7;VGM{{6T#o+}YMiec~>7_=tJ zMeE-upJq~*^exCN(q-$mFAYL5Y9yn8U=dNDA>OckCXOkyxo*?&4Y_(^&=LA|pNgTbUd{ z%E>|?ABW<%$vv*7P%Bq`O=g8HJeLZ5Mi48{uq!(#F%oMj15bV;s0E1m4C-GtXB`iF z+Ij*IYb!9;wL{urY6|E;pH%9G%X#9<5rIEyrg9X+=hN|Z{!*L@G&!_>fchRZ0+*J= zJ$N1C-Ej{jUUvv{59KUqv1fJ2yiDT7x_nl#mKz1xwtY<^?!v!f_E+N(s}}wJsCNTu z)^nd20WWCjSCjLCj16z{ypri)y%CVjRO#ft*ZnhAK#z-Y05OdJO$KQwhUyDDL z$5*jrhIO$=A?}jjRe>m{ofPY3B0-;4V&A1?CC7Od+P<)HzUOh}Vk$Bds+`jkk6`lt zGr)g$@}5VBYC8%ZVK!nT48_U{5>do?!DQMRb*s|^A>tThz)2!-_mCr!78oT@SU3{1 zfS@$}t%=-O78T1)&g!yz&(g(4?=!Pyj1C6-M=ov9aZ&%U!ml5B_y6G za0P!^RLDN}DAV^9BwLbzC)~Sp-hSlcI`rLO|wMVCGA3 z@55+V__jM}cV9NcK~qsXaE2?8+!5#C-92#0iXGoP$DNL5h}0iF&t#HHzBA)|iWds# zH{e_dQ;#OHG+89)sxl~uQ+ii<~h!%W1t3N3M=st@S~B? zL{br%8R59>jZ^G_V$+SE7<0S+o?_MhIzjnELQRZ)5VJ5pBdLQQVB)Ne(ch8$_?zs@ z`ovRW9Nn6OF&V|g-?G>Ld^HZh{hrJnMjs+;dJWI|^Ar!@3QFH)(=wnr3X1;(LN1WQ zgR`PccHxPn;>H>bMTLDO#Y2_uTZ(VPg13lCWL%rE2&p$6uEh&rBH&Q3F;IO3@%PU{ zP+&$a;w$fq@h}kbCtq_r zS#4<=5nuchT7p)N^kRGrWCpKB2N4n>CVgo>Tm8Fwhbb?F@FyMqsaa!qOd-U!9?wJ8 zv7qA7OB&jT!ka@l!q&?DOc!E(z!OOTYJOvn55oIwdU6jMZQw~*WQ!qMI^)i}4;!Ul z92jP)*s6dctS%#_v5F^e#%Tp3?1;e{O3Ni~wPsMg4{w}Cd};<#-soRhvKBf|q+KS$ z`3D?W#oAD06+tgq^Bn)3dKMV|3B;*gHbhAzz?2$g@%~*5aZ90&4|y(!+;|U#6bU>| zN$C8|E!u`!!rbHb0rVLwv_~-9E?wQvMxOW0jrpKW;8qMtzI0DUmE#0 ztp^mxLQXD~FB`xKF5q~?To&S0rI|GSY7YdQ%|2)qRnj&0_U;;^!wHMN61$M2kp$7_Ucu9HF%IUcW)4Sdmn zqsQ#cYg4}uHub(^$rWcp#f1KxlRwIk#&i`syz3|Xvdhb;;Nd!u{$7BjQx`Cp6di^FH$pXw6;#AzFpdaX@V_Wta=+)i z!Mr(|IVqs(4ZM)jO5=7M+fjFH$Mh6s%F02ylvm*w34X*@?EKpBgBzuc+7`8lj2t%{ zU(Xf{ltXnJk4$dzbe^VbXA4d@OcUnP2E$T9p$=FRGd#T6HM|)WFoBq!hETG1^G`wI z83xl_8Gpok5E9mf!hLjlZ#DkTQ6_t^t&+_LntvvHEY3ZuS&K_&J)J_tPv|$W1*g5x zmvH700su)D=y1%3S!ac~CKjpyUr&#aIra|GxZszP%uYJ9Wc(^%p(j}aL8VW!!^}}+ z8n%b>qqaqJ1)V#Poa<3r9G&X0j;MF5u0!giGvHFLNm9y5wIWHhTDtq0f_^V^qLVpj z(u9?}v4+&?+82V=7Cfot8ELUDnj0r7rfgSt0-1nEz(rkSL{YgtOfR(-V&=TXjyH`! z^sT@cnxZjX@&>ONkezH3@!bqBwW7QZeB(LSCqlh**5m-ALj**|H--?WA;+ui;+v+~ zk!ueZ#&TOAh^Xx9t8Kh|H%@nAQ5&R|?|BwkEXXS*%e-`P9 zA-zikeUh1tdiK1eZb-b7KRBb|VsV+wOqe6>z(LO3 z1IjzXAAYd|pz4#p{^8HqPOxq;GiHOv#RY;q{Tl?zn>m@%oz@3a2tpM&gZse}egon= z5^0b85=2Cu2+KX>CQO~UulzWRRNvGT;MVv_qEPX$3X<#6P(^cTrs>^8y*}d-b^AVA z8SIYZ-K*V4hW_->Qo5jb4p(j`Q+K>=<#WP-vO2tq%@pn(8vIjU&A;q>OJJ6zOp>%( z8eG+cpu5xH%hFs9Ef?)8($s* zK?}}q5W0Z>AZ)$Kq<1*CULeh==OINBZ6#gAF9hiMBLEWb@n)|Z(V5-DnE#&3%#a!t zBYEL3@mV0%9j_yUNmJ<8=3+Lkb(IMTf)=eX!NakAOqbYhy2!+OoJw#AY}i-6FA-z2 zQxYRW_U1k=cPq~>ypTcda%w@EBjz{We?Ff4uIR?Y9xjA^Am0>|aZvAHlpY%-)@m4V+<{6>PNlXjQnA|f_r7@2*I1)oiH^{%qNL%TxVC7W5A06gdf&tF(CDG zp`BK#G=ORM=LY1L+$GT4!W5L}EutzZ6l|C8EAFLQ#v%1L5NfN)f1m@=?V!v2xQkx^zkQ zrC>657@|9(1yNg0n^p}hqpRj{rP^kLk0F!Lv?t;o_K+uEsoqes42gKsziXlCXFwP> zfYlP<110mGnTBuJvu7%$Q-{CCFBB~Ib%#PmmmdC&DuoA*-e=QcjzK@kWsLjBSZsGm z?DN0V0UI9)N?BBnP%a7qXQ$=BRDN~#^?b>*+P7~N|5z5knZGPWp1g1?HbKcHGLR7W zjdAF%f%|tK&bOh3r4Cw*+9T#=hH0@mA+_6j3gAbpUV`FD#@}!6Dk&Th`+7q6c=3ZHRO>l)4Rc*qgXi%NT zM5S#q@4DJOYbxlg0il}Gwx~fQqtp34M$>CNu`_*NJB)tm9=|1wT%sbl^jN}dV4HhH zycb>{R>w2rp$icd{7`n<$zQw{e6ZX(@W&d^2^{ zO0uWmo*hepC_T=QczXY8qE6gYW2yci#282_xZr7cBm6_C zBxIChjF_$|24P;NlA3$Fuy*BHH}LfsL8%FGEV`BDdKq8UM9P!kGeFvT&R0-mwohWq z15Zu$>+QYK-^2=mR@XUW5T&$v)OEB~FiNdyBZie{p=uZz^lXcGriRK9sYj$HZWy7+ zmU&bY*;G4a>o8wln4c)$0Od&;VU}<;{is82sk(q1&bPWb**?KK?6p!l$(D-ut@u_k z!O$@J0RZekg9l*X0`&U1Gvk-e0&WNhTu%Y5cf3{n0D@@;w@*Mm&-d=ts~@Yq2Cm~H z>;Wndv(wypW`fhT&s$3PpRd0a=DCr65gy+fqAYZy0u+^k&RemsxW)wY zoMFXhQv_j?#u%R;ICGaiz`MW)tu;;B*H-~!8OmD3A@AFa%IO%-{UjnUMOz-#djI&GS5sI{qoa3%76Y{xKP9n zGILh8m~T}Nl(xbgPB@;_Se`j`ovH6TY5ygkQKlEoPdQwM-0X09OA@#_#A1p0T2BUR%Z%ZUK9E^{`D|Ngh?Av>L-l?$_VIf}nkS|IH9&}P zj&I29JVs2&fB!~RH^ClD%NA66QZTz@@`%TUbCDNtA;R3_gh%{_A{&Dkf8#IEFCa3Y zB#&9~{ZPMpL7sE*nY&{I$@U9~>=W?%`bszrh#iH*`}uJ{K$T*vkEI(wQBS|m>F@4U zToVU4h!$vw`ZS;c{5f1_pu@7yy4}G$u1%=)^)zD@?5$8HM&$Y)*-T(W0;HOAS`yQzYz2M*~hJM-_HjQpJhOsUG1YGM19 zG}Vul0(BgI$nz$sqFq?_1WX@?1-#AiID6mYLrC92nDip6_tTGf8B1l}YMK5e1dbG- zmMuf2TfE;^RAD6^K`ewB_Ixuxg#Y(3K$H>xDmDUv6x-PlNmp?oT%=3I!?($j0A3v+ zm>3!OuLOxf-(rH({brWZn08!qnsCZu&pHb%!M#cdN=gvBvcTvMSV7KSOqLOdpi455 z9)t10$NCeVJnz&A0aUN(oL)|QO`lHWpqfl^UQXadTqtgS@VZJaDEv&^tRE45!x*nA z5e;+05`z{+j%Ms~S(XGM^*&QRyS~yg%te{nbbk0@zUks55{k^HHTN+8i42Q#c&h0` z94Lv5aV3Txw&=~);Mi%rvy_jd!R;E zpgy!6I^6MO>dwd(65BN(Hi1Xi2!kt5-320P4e%rOB&a?B_5kob6w%fU3M zo(}uA;;X+kyR=oiwBs~& zi${a;h;S`4_9f2g@a+o6yo^yT>(WDYd)B=EY>5n0B0F6t=~xrpHDsDDU18{AL3;-3+4gyfa$%C!Y?D`13{n^lCiqfcJHi ztzUY148X?w7yG>U)+-;=@dc>ny*~3pCgTmnwe{IfT*+-lL4QRZ+yaarMn+|{zw zq3eHL?uP7Mc(RQDI1=+!al4WhEXVVe*|af%-WS1Tt>h8KH9wLxRu(=oVFDs7qDP6} z&z3Z~#9CE~A=bB{%ueEOMU{<3c@5|8MCQyA^vy<9yxrJbTJnvE7aHt4&AHd`x_el7 znDp(($j^WA2(aOee)5w;GS47%o>}|VZIEhP@{Pg%O>{yN`wxabI<0zZdlYVvZu^5} zvB%d!9luCe2RI~)R`iaP&xf>ijA6YxazhK4xb%`GRyL2L50+B|BmcBq&~APGv6I^M zcSj7ir~F-kd!koiLjyTrb$H_laCqm=EByfYQfdXXCY$Sh=VF9vGqAm=fy}Mq9(3Xh z$Y*iUd3JOd6)N6avMx2qD)oUcSwI}ed#g+R2_&G&m8JacbH<`4Fk>z!zTih z3Yx1UPjcz?-n6XN_N*M7)K8aq{Gv|acWK^^q6yI2>q8%CEscU2dJ>yyr#R5jly*$W zncz$}XJ8+9%_bx@w7;3t-r^IR>VC-TYOh48cd~|53KyFZ?CM%{j|yq6l$IpfZpKN% z$z>sZn+9a|-KpX&eZ^EDj@h%i-v>Kb~z>=DzC5Lj_@V zlep9K!is#Sy`pfphh;+9aUU86z9Gg#`0x=A4Sn`P&0@$!gT(%9%?s`Ga_V5~Rju!!$Py(=oOWO|YJadh2oeRgMks^k?r=OhD2w z01vO^K>!zzfWYm0*M_(9tM@IGcLD}mUj+%S&Bw1U2Gt}&pJ10FYrvevcYlk@!}`ZH zQ{KEoKFz|eIHdBu__udY`4tfmCfALW?62A)>y*EIC#eclhtFKud{vR(aLpaK|}V#Ql6_oNMG&HI+Uw|Vl=MLz<(}n^|4765BXLgujJgUH(*?{utsN`1Bx}i zDRO>&`2o&eN$~&zrXnrc>F>Z^7W8)2-mdwrOv=aMKC2 zBhq2m6=qaN<$TaztHQLNnL+XAn*=7r^)W{29Dh<4cVShlqE>Q6uEQgd0q znx`s6sw%oR(89Xy#H2ZE+-C|5%2pYL!1Oc>W@W+#so+Mr&BVS|xb+{G+N-OF3zby} z+lwQYYxV~7D%j4rRSJUQ%qamay5)<3#bqj_zvgAI_Tlm>nFwOu3mm+IjMI&3JI_^<%&JiAzwNe*q5X+cp8a6A!nHp9r79-c0dOvV?;%>O*}= zig7IKd!6~@d>C@|1_3j_xtX|A9S$|NpHg6auIUUGujDsN?KOmE_#$vL-2d?VZ(I^| zz|t=6{OAZ1`7ydlQ|OaQL-^uLKY`<)QH(W#ixq%+&im!=)D^ZjA#u#pYd?zCVhM+W zt&N7qh9-a>hPOeNFjvP<`@4Vn($CMC6}xl=@cp`1rvun@SAPfivN?9HA!4G1pB)fn z&_bZop^AbG5E?q{?QWKNH|F(9PD+Xa(vRQHjQp@dfEO}NrnxL^kMwQRM4PcHQ_||t ztKL0eDg*3tf{OS*7idF967`T2)T)Qa@aQ{5=JC+U01;Re&Qve5t6;g{n4`}aLy793 z(`|DkW+(JeUo6KO4KyzEm8hk)YP0ksYiUxMEerUcp`$)U3bypw{j413Uh~seao%O0twElTL`tDUSta{MExeFgV(HMIFS^2?NZT zvgOQor>Cma(Jf*BcxUf4PpeC#4l?iYds)dC=HXc6Q=ff{w*hA#xp#P(@a7@|IEtWk zxu@I)i0qkzMWs4DEBF(#vAy0{H9|Ti9Kc2c=*WBWs}Q66Q}|~OTt(jY=wNqWCVGyYJGsV)-B=Lw}x#2l;pnS z`ILPp=yV(*Q*_!RV`l`F_IHejV>g$10FKLB)cG!6jH22X8rvkVs@^_dfc#y$%b61E z1m834^FzA&pHuC8WJ@TT1}2VwP6c#_$7xRZb;@r_;OjRMyRk5tl!q%;^pX*Ikv>Cf z;AALCz3hpm3E9Y$-@&hzFn%sM7Mv1uhYx^qjFUs%}xuzP9! zHqHqiR>_mhMzk3YKnITO<~;l|@q*(bQq%b7_?X^0|EEYm zkIbkvel4lqQKHK}z?UdLclkQ$gnY?}Tu+xuon$fHfD)fl^?S*@|H1F6`4_X0&MYnq4|YESHR^Df&ZCWV8jICQ3waRmIgz&pWdz0z$@g?4Rm+(=#*!= z=I@%>s7{$sBS09{A)wFuTSKkb<^4H4(@lNeQzvLbryq*kHZ;}At|nxcnLNwAK-2MI z)KkmpOQ2OU_p|i)HMcL@1Q8--2oU|UcnUximw?=U>#gR!yeaP)jtU9T6cPyj2X{Ah z*9o)#`p*Pub#9-Xn(~Zzv(G6EVMA3lcAe4J5%^d2Ri?RQ`K^4`s6M;F^dd80^};|I zBIy=z`1SY}`(pt6VgS06Pn8Y@HrvrVB0sE|{w>n(u^$SkFmj+l{>fN(=C#o0A#=#5 zd!ls-=JD9-RNK3rmEI@1|2#ajVQ;h|p@j%O=v+BLB7Y}Id)x^d;n z)E|tlwJ>N+pLR6`#pq}am&JMMS{*e1y4=m2w;$Homy?isK$?yk?9D|Oa@TZla!;JE z^2&GbW>?q$XX~m{c!t8m5cuy_=IOci^L$uI!ol(bVJhF`XKlSCd?J^_eLi6#!7#5VsHQ_v}dxf~)-rWWLKGrE{|Fz$~d( zWz$>Odvs8Cf(T?za9}Z?D;5MkJB3d?2o@N-J+U4_dAfT2Q5kDJ*O!kGyjmD8u(Y-0D&>m%{9J(MlJ#H?r$Z#Xw8JItK9Lh zcvT0M$`o9EOv8K; zk`7|RN|lQXUXS-&)S!$@8!-US*6f+zN1;66^08MQU?l}et@D03Z1}PwJD0zDnw==B=)v62TJ@4G*P01>jy25SZNi#f*H+>H@sTXBXA` zmA$~7L9zoJ(XAzU66yNlW}$}x3CzCsAKg>HxYuguhRHGq$HSF1E*ne|y%+1AvbA&l zEv?!Q7ig9G-4z`gI-Zl_Z0OjYlDjGJpr0sYMuy?BP>|V7(>aECYe?C@*6p}<*&3etW3>%(4hh1Rzk zV}c5DJGZvHLbx%Z821)kyl3U4wsstDcVM*csU0OUQ9T_+vqd9ZKEKI68tPwlIE6aq z_9$?ZIp9qRzdS=Jhv^%v`9jd$!g3PDfT%#a0U(F{uNI6Kv7R1jXK6X^)cvIz6{oen zyHXW!<;rI)5@`=MMysAEUEhpSv2zfb%22noodGb(%c9yW=0AyE(E97}{u6 zO{P|!lL=9s=1Sx{70!6ttj_@48nlEhMGBw|3#ck5@WVWowBGGfy$^%W4C7RH|JCzuMDha|G$nj2EvR}OTfyB{Rb3}3eEKWl0g$hWlVcrO_7a}oT? z+7gS{tkreaO0PE%N*UGWooJj&kT`Z>b|v)utjO$4xKr^PFG`!j6;qh<{^eFO#mkgn zi>lvrM;qQ64qMUm(u&&_Uol`f*7HwLh=}TlI7N>N1wHcYemK8E8`=KxXs+KRbKvnF z&r81KKb{1-WXNu58L}mGH{|8&u~YK_vKYThK!PUlMZ#YbFDH+yVkH$(-RFezUIcs* zAD@}AOvMW0jqgk1(G$T+97Uwl)cUGfJrBbx8Pcppj(f+!s%9Afw4zIuTRV#N68=69 zSty}CpdL??j<0iQBib|gB2Wui#amn!ry`I!A&Ylnj0c?>^nGn-=y{pGTe`YdHKem4 z$%d%stT!;zUx|SGlXb}$DtI*&EOuy~Z0QB1vi{2|Z&$a(mY3m=FAEI=?#D#7@%4va z!~CG>cW&X~libmlSupSIB|GnRW^)egs;-wvxxZQ$d;k94C8-^hXf2N)- zxcLaYzCW4`Tc(Gh=xEJ@sCWwODO`AG!Y6pHR=7oxG2C=6;DIiW53Ug-WniFG=#svH z0eW%a@lY4yzSzsHp?oT>otg5#?~RGZ^(lc1sz?AI7q|SIAA&;1!>$6=ckA%Ot}uuq zfx}SGHp`x;Nu9u?D2Biyt>7lS42TOt;ESVt;7fN3b> z>|`IVmluVMsyZKg=9?H_@x4GJz{$PAZ*CkQ|Lu0)yP4uy0Mpl+0c&=sEtm>nmMJaR zc}D;Y>$nv1>&=gGW(Uyjw>0c$eY`F^5kNuOuYpTMFBMmZwVsPR{F<_Oq{%MB2f$T8 zJvIN!;pCu>DWa_JOCm(hg@a00GC7;thPrEnI34E&Q!@8kax~ig2%l4UUCHfja#MZH zkiwMj^MZLXKWMDf;br%Qvkiv7 z85ssYaRi@$#H$P0KCClgpSr%D2r%VY5MO@qQ3(l12_7KXbncg)X?Ci>f3-cWU8KQ) zA53kj21qA2^=N{{rhgM=EVcp4FEV& zUMo>RV`(8W$Pg}>LtJJbyf=~o?W?`&ae(vJ-mgLPT);y*@2tr8YQ&hF?p=qHP5#;U zhCJQt#8RLu1L+LeQbT9h!;4hMrc&#sekt>l<5bD?i+fa&1_`COVrTLc76 z`TU0Z_#Fff2pVpC>bv|kV*~rKy|A!?c@_A-PJbaLd4R>P;X7&lmgLozI2vSQQZipH z;-G}T#Tn7K|2&}?3TUXJ=vp69BpwZftxJ{7I@apWN4f&#`?6oHbGw0$9hRx546542 z6xltJiN$TsIgBul;8G}S;4)}80nxK3p-+&;c5%{>Gq|!9!JSQ%@dEYusABYlyuPzx zMGU5>3DT#?N=vt^3T)j6eiU>KA!r50&F=AaAd?+w+~? z!Ll8O-89YUj~^Z3kV`%bIcm7VW^GKF3?wRr{2PHJEnqIzN;x=a!K)A#l^ zjz14ikY4~iB}yUsNwGe_WX&pEaRyoXHgwLdoBYH!aKKHcDOD{N&g==pajCT6a2l5Z z3A%kIL8Ujq#0Qj(+AaJaV@K=?>g!S`&dal)0_5$}Qn4Uh=h?P1wO;EU=w1AD2ejVb z_{h&=NRc&b(6=@iwaw~ z9grRdEEQ}wTWGxr++`kro-}gQ2trp)T$rVrkGx0{(ezR9h`PAmpmDW_jkFG#%39da z!qtpt{6pjSIT>*)*5WHRt$iP6U*(?uh$Z=yQ{Ha zdJ25S>z@r-_WtSbj2uz1JZ8u1VHQ`|Q)oL`IwWFUTv$g7h<)YR z2W$%PU;P8@%>3uG$QSpg86yHu^?@-TfgA*JodWaVA(*Gl4u;n;IYATOY!8tn?t>Wn zy{2&H%LWjw?Z(OHExMKQXtwQj8u>o=MudAVm9mILpGIIZfHT3M7j}iMY+j6hZb)|j z_HirQ#5fU|Fj9*W?i%2Xm!}4xJ_d(fEtiMsPGv5kl#ySsIYptBHqY}TY<2?kKRNnf zFVscoq3x+4de$=}L!(Uj=HG9PtpYo+iQb_+f=H7~`h?rd0YZLo+$EmyD)(Vr7*Lbu zJUlEV8W1%XAYAW@X(S9&-l?PB7h$@aub#!5mt0IE7-~!LG`VvcnK`_GW>G`0ywbLU zR~k6oFAYQf3Q+FuT^qHkmd{oW7V3WpDPw@aLWhgb-S7Qx2e)rO8U-C5gD=gr+GF;i zsJ~(Q1?9o7AXzlo$nkFfYNje}I~F@X z!{OQ_=9fvTM9v+C)n()qUCdI1_x5l0?H{5Hh2xv4u)T?M;;GyiDa}^H^P$t-*v-Y& z&|1HgzR+yY?rA6IFgOOKlXkJS32bddP|=rPO~puTSgdFP8-JvQIbf_yQtAgme8^=w z{Ly7SX*hhEtU)SGG#Jq(b09l!x@KP-T!Y8{f5?=iw}<)+cRK|65&l}NU%XT-h9lz6zVJ4LjwVmWpNJi zfOp59{;UqVN#o+S>q(0~1Gl#t8zj;jBE6tK`g*?9o7*5&meudr2XGD}O!m7^+bqA| z3+y~ujRV~58Tq9@00;;j9|Au@<8$u8FOxz+-vKZoFfvA=;qm*@bN@{3F)c1Lz<|$`{(sV@tm$$(7lr{CG2PJNl zC>1Qzr!_R>HeQhA07ggRP+K01niN>dQD9>nAxV0T&%rH>F2PnEuPGE=Jy2){Rl|Yx zco7$}JL1<6V^n%n<1dgiEZ|8xZ)>liz3_xkK56NtHLO--8%}l*-iJdAT;0V(JZUvr zYFd5~XiwuhLYOz58V|jRvN}uynvs9a<@8qlURL;5 zJ`Je;ZKsw9T6F^W7?l{c1OWEi4gs4U+WeM8{pZ=!KWtk5v%xl3;9pXq7rW)rd-7it z{f!=R8c>zaI{}1-^|u~4@pr9F#ftpy`b~I><9AnBxjy}BnP}|QiyRgd3lyrtAnNUp zi&rH-M~}6DE7YjtS{~Z5!&=QX$;sS1kkt3=hYQLjtxk;Jr#3ZbjViw;CePw}bPb_j z+>#3gnBO2W&Ij*KRxQan2|HVJZ__eo&h&LErFiv?fwalaZz`2E1-Ei9m0Wa5h)Zv* zP+SGK8n3Qz<<5A(%aJIsG>XQ%CU#^5nex*ZX8FcU_C%sgPE^(lD1(!oNUOE6z5YtF zr~8;PPt028@;_y)e}>a%m_9?unfAxUfwRx|W(kJWxGZEABZbzVMK;%#2|iyHfE9d( zq>hoyi9Id^rUVi|VpeCdJzVetYea7of~?FE8>^pmP$sn&ZS`Na6y|j?SlK*9D2+96 z$<9Q1`@nbxQkakY+<5O_6#o_{B$P!8D09#*h6rWpygYT1eA}aiY8nCU@qQDqMtF&h zp%GHaz9_h`pD?#ZO^xI_>~x3tWmsASP}OJbU}+4OB(HdUb>cjzAYh5*WhL*~%@mSRVcKMJKs@^^SO~{ju)eB|JgaOyF0C zKW{B8e)?1Cy%qHRU&WRFOLON?M}%~mCf>Vx+)7={=y+TTR28EP*vBdZyq4j zQ}Pbqz&!l{SE5qu%D4uRaeN1YcJi2hEIR^N zQ4odwZRC2RwL8=;@1Q1<7Z@JbTAl?b13vjfkHtyx**Y>lGKGqI#2Po^zYz>Wsa3IP zI=v$cn_;3a4%JFoq9v+C8D=T-u&`CkgmW)E9mil~L$Ohn4M|lsPNW5!i|1Dedc5^s zGO<=FbXk@GJt)CG5#LzEYM4(?5(Sc$fn=K%hQxw*1nuf| za#E}nHDztN2M8+~{^^ding5~co1-I%nzzHv#!fc2xv|ZOZEUdNCL7zflZ|cL+}O5l zTl3An@2B4%J*VpQIny(@yZY9xdYXOR718?aVD3 zRh#WqdtFeua7}0R!EskJ4JJgw`CK=+g$332h}&0KRxqG#rvAfI!NF*MxHS zM*>gw7+fk6KWRYeRA-!4STJI|>yg1O@wEfBk1I2u48~Z|eUn|Q_*hh~eMIti+E>Z% zl!QO=U%ZanFup$)MP3vl&tM825S}sBY;~(}7_aa={CNg4El)LmM>m6#l1aCJ(!?mtZ)%Y0Sh7s*Cp=5T_#?D%2MxiW!o=AO^%S3oaH1*@a(I{|2VS8SPZOD#$a z1ESONhhS`fH#!I9@3n-kfouG5s=nSm{ud^A@wq7w>ylDtp?KA&CwO!fLxhX}4_nPD zjU&~)Ndl;isBkkfw+lJu)`E&9oql6tCX9^vlK)wf5+Y4|2lIRnklUy8TJuf5C}sIq zzRY;n=%$jBd{YkA0Jn=$5EdDbN{p!zMVTtrAQ7!TKf}+fsQGY8?5Nfwtr`QC?}LH* zEHG3ssc+fxJfGc3M|3+tO21(Zx-VHGO{gnVTzjG8=xNyw1qT&m42J|Sw5iXy3h18( z+&3ZKpuT$u4sh#pM%o29rJi*(0{q31U6CU$TD0l_U`XRRS(2B2qSUi^gT73V{a<)X5%&dL3n9#X{%2K3>?_8V^9ri~&VISFP4E&GEj^r-LG_ zT*?1a+3~Hpl7f=!)kS^B(kmUST7yK2#+~A~v=^(qf~mH-Y$(8ier13hIlR zISNzffHcARc?DYBseWKKV}9`q*?-8;&!_{vF=U2Y=&OSDBr3!RRYbo`V}_^nOFKJJ zZX=LVRdt**ZH^oCOcifGA0KaVs#wkYc85VRdfu7qlV>erV_jd`}wDmrW2!h3RytuW^@- zf0o*D49|3DOV{B>XT*Xg+(*-;k<+qm^9@&uN^-ed*>`v5-6oKzko5DMaN4ZJ>@0X-PE#(oe@!u#YVc~f6G=M z&(gQvd-S+lCpA@TcJN&HD-AejzPjdu#Xuau(?c%BGYSxa$MogCfnC_Szp-cRGT+SzhKpvKdpLR3rh;qf`sC01Y8o}hQaniE5w&s5D-PnEwA%golldPw zm%1%pO9VRz&gHNA*mx53lO||YOmk=?4TiNSDN^SGb_0MqGM53#t<+(&zi91^9<^X@ zW?tcGc?NU|wLcePBCe|8NrmKBHTsfx41{^3viJZ#Zx^%!E$Ctjf!&N@<1BLo1|~cm zcu&q6l)NnS_QLD+f3zFCS3f4_J zh;8*J0Lz_DO1(M$u*3nE5{>>N8Z?YE)mv@R_nN>sW%8RC<;9Bajo=j%g8ueA=tn~v8 zm>UwYLvtn@@!ZqeLY-oSoAKR2F(X6_rx$q5*8Dzirohr^y)tm@Mf2{s+7oc!y_aD3 z_&N0f>g25rChP$#zpWFoIMswU+Cw@ZNQ_{_g-G=Jo?%4f z{%lb6JJ0<#-|6MEJQeQmRe%O)yAcI9NMQNkpK|;v79ibNhtyu`z0{dED(4IDX)*p> z9t*k&u+>s%J~!xI)APx)c`dEAq9;qai0OH z%OV?AW@x~dAMl8j@0c%RdN}ahv+T9zrxdm{1yjLb=WbP8l~ysfO0RgZ{BSu^NabL^ z7Oaeb3r)ZW3K@n~;9Gt`{{2gLLcR^qjBYA_6?m|#oseD&)LG45Ngmf&W->IKXTl*| z3un=HJ8sZhiPF7@n{(e?;<7JqxSH(!WZ-u?=!ReM6d`u9DgiUS_36k?28M)$giZ`NAt|~+sa&;N6w=0f;e?!K@VI(X=A)uQQV@OellAI;xj?Wuv7B_JzV4CfiQb}dyK#{^(?-Z z?D(yu!X}y$n%w~6(Aj(8BT7--;MS^Mp8Vg%T!9@16ybc2kX`H0olvGv`*DjPjinnu zfem*|`C^d1(Syf}huWY_xZb`HPiV^IX$2qm$WK~-ZGvi0mX_(Cf{7cuP{E4lbeiG6 z*^=V&n7pz?JgjX~#mEZT(tgU<)$X`-ZuF+M zdR1Pd+!-@%Vr8*EcYpFkGA2eH30hz!-vB>u?9zZSZSU1hg5!sYvA~*j0g1C3y^vMC^;m#7Y$18Uff@R8N;mI z1b04shfGB|d+!Rb7Jfhl{y^8-M*|Arns*wID60JE_KFq3Rdr1dVhfq@ejZZeCKx`ys5|M?l;Gl?zNjws*$&i=sVjt(2v>j zo>F)78NKU{iXOE16x_VJ1==2<%A6MJCT!nNj261T@ylr6y9<}C+SdN12&AEa9}!%5 z)1!M|qFN8{5zdAb;m*5dd39M2+YcDu{saB_r31+;xaFhDj2gpk=wgf)l^q`Z;|t2H zlS=auM!t`Yz155=+$Ne&cMgzxk4k#(m&#ShchV%B#+=#u^GDH*i>)ala8oXL*` zWeR}2#M|q-Y4xinP?+*{He^H3D)iK`L|>z6DJ&FXg47G$6f1hBVQJaHW&hB6f5UsL z)q%6R(BkzJqssQoLKv9f=~emBZRZTEo&z@Bd3G9!#+y=c5iWG3I80wa3hWd^e-aV% zP&k5n!5ql=v3&Y)2}oZ7GeKj`8`F?Dj)*#I82;>+wUv>cY4XEj^--sZPgZ#2k;%@V z9gw(~}YoFaJXRwqrVCqMA0BAZlrkh$S2g28# zKVR+L8tgi+0_VK7w}DTAo9<9;K`c3mz^md9LB02_rmX4|AYR>hV9y?$pE)6q^fgp= z+xynJ5evbd!2!?p>~l;5&_@evf)urxv5*w_~OEOCD1p~4)!ig*@nuUon+fG5gQ8KxOG*WfFZC} z$ow}0szL{TYf)RVUf=EeI5RLnmZ$Sf$0zN~XCzQdU{_@33g%XRNc$0Zy#LJn3rLjV z_y~eOHcYpRzg=D2d%5c`sAYG8^QScws0?Po4GbdViyYRN<~|5GKV@-2z^H&N)h(Fs2ikDg`3U-HZ69y!ck6v z(cb|x@e7uK=8nsiAkx6&$Cbbior9~kM(3Q)y+Cu|S@pX)XFr=ze;VNvEvzv*k34%v!a11*TBQg%%(t5y{@%7E(=DMNU+eLxR>&}bR#|PuVCVY{<2XUW4(p1?6F8m z|0NUw+76EQXMP0&jMOrMbj`FSCe>nTC~wg~3ctL^7Fveh$$&=34-o}TQ`vojB2Y3O z|DuP3sX{`O0PBx26SI%?B4~u1>AyozAd34c>Xo^`Y=LgEJ#8m-d`E(fsnJeH_rwO- zdy9A56CuCH9)MY$qUWHei4J3Wr1NX5Svx?&Jej9N&VFIOhtFKvyDe-?9rRTF^y&75 zX%PeR@-0tg0JpAgyKRqFWZ!)Pu0N=1BZ_V&H<)!g)fUr+!xx}l5MtRc4i zAVcgCA>KfcL2->{u4mpn*ET+A5#VCBxY_{L1^CJ1XI8Rj-GU{nOd;RLguHfOy>cn0;9m*F0ctZE`^qOLtd&OCo|ie{ny8QCr20xL+(?Yk=J2AuSjy z9eVM>inZU$=^I$=#U1jpORXSe%w0d(r_|SiERLg0*JCufau;hDG4eViLi7XH`qs>< z+TS47SfrYH@yZUjeW3o;^iO6S^2f} zPV{0t8YeNArBhyM?o1$BSz{i;fRqWI{6@b8LK2Z1W*3pg8U+HhIBlHG;q{sam|4GL zbaMX(eKrZ(dXeex%e*-xus418IA6{DN9j1v`lc&_uRkuj=r|MNB|a!RAU0y1pIKN% zM9H8l*+sy3SMo7-2e)`oJ#>(ggd+Tnl#po$yov<^>>HFXhn~^PeXe%??6*EoH+j!! zfx8wb+*h-6$Qvfu9awm2pVuf7ceug)8R}bh_@8}hHDff-zLyOrP`(Q51}XB3%Yt*_0qM8T{l;WDZ9HLCSFJ@N14pHA4o@*!uizL9v6dtX8B_aXW$h7py= z$`ev{ijVdOo@7F_<~|f+>%_!P2AZy%y`lNztd)q-nz#PqEpaWK-;s5?%#@Edw*K%{ zxIH}KzXtewJ-nrTEO1Oqv{zR8SpvMXEDfNGHNP{dj_IIDo`5TO?8yeiFv2>3bfm=} z9({i?hktiNmMiQ-HK7U{8zv=RwwiXyA(x`x2GC;b3{;h zV7hPZO3NU+Tx{SW)l{6nNqv=n&HZ%MN81CRXO^bZR~TBfhV7F#6KDZ6U*7<>WKs>V zKipTvm34=5UB!iT<|r0kxO)hINl8@w|$A_S^sqB^%-y)EhYccwFJFclGS$YmHN1BKcv zf0Es3)PbC-6mW>S4ImUFc$g&^v6;Nk70bQX`BaY#$2SCbU@GmN5RyL2HPxRUD0#4! zeo5yO-rXSh`-uC<3`#&+sjPCVTfx`~#O;d(#2o=%8RkqH^5}O!*2ZlcLIEpF_WYbG zFhRdE9L{j%b8E+L^^+I>%H^s1*FkA9Q0=;<3Cn1XCg=sQ1K|I?PIuq=SEwjN%56&z z!_}l!X!LFqhEWWDm$kr$1&KQ?ILrIT07d65^Ir9Qb{SrE8rQK5{~f`C^-3ZN`Mfm| zS*PF3?}pQJTRt==KEJR1Ep%Hfxt99Ptx>%;>P>?Ja81J91dPg79*qeRYgFge)6RMp zz#Hy+*fV%WnUG`FVnqUuQuKGNlh2fYNrOrE7_c>o$`n|N(~W^H5R2|zNU2+s&9$zq zHox05g9Jah-zYJ;N_|hym6uE*w;ekXHB$C(|ayMp5Gk+tG7Mnh?KE5qKoaY*;4Tfn2 zB=!)Cz<+t>Z{Ibdg%Y}-Ql_!qHrA6PJpM;~(??LpZi9nNZ8+;oTf>#BQGIY!XiklQVG&hMGoqAUOSf!)-(ikaO`fX3z$JUNSAWpzXa{k9nX` z(4xtC72BHdS07hff%+oq^JzUGa-bpJJDrP4k3iNa8BMT3v>Ywmp^q!#hxvkZ2g4q< zx*XW6&^sSH>@N)8l$)&A3LGDRAFrWXbhhD6*vmSSCad8Nj?-{t%~u#Z8dI+p|Fa zjD1r?#4L~NQ+GfUPi3`l?WS*dCHU|>yH6Oq2sbYdJhD?r`1;gvp{rzV&&0Ww9Oe7* zWL$or<59Ne18%ovB=J}1m=FYBuc#xc83(#}CO!fQ2s7{Rf8Xvgb}3sCbOL%eT!+;$E{WBlat+Z9S`rA3kf1U( z>4o=9wWJ#~-dMF@D_ZlxC>+VpseWlwUIXafGGmMmiQ-Xv=RkUtgU=f_eNfSSK19C* z6H4edGY_6{|Lz6N6;c}hv5v+l>*S+7oNy1JYPOdXAJ;50mM@zQ4DX_7DpqZaU&guS zliI6i(-xt&o76VS=xT}aI+5qwrlM9ta|>DHO! zL#H8eZ%y7964QNoEsXPaHK5X1{fth)tw!shx3>Fn!Ruk>G`}HAP3&c<3)%qWGvy)@Z3PHnp#&r@f;qnuU zWR>6K@l=z-NuYY%mGqSNSCt#>q$yOtb2!tE0Xk;@f31zjzfrN>{x~ISyqRTQ?Fkun zu+V_pg+q!{sS+rrwyoeEDgkT%el#2~L5m#yy1G>VsHzFTC&sxFV^hpgTWCVq_igDu z*2DePKFD-hg@&h&V-u(j?RMzk$&qP9bZ5#QHH4x58ujV4syfn;voP-_H&ItB1hi(u z>f|o4nG!Q8>>5m-5EPY-d*diD(&BO04<>vxyD}39)9-$Z%uv*RR+1o=huDoSf%>fLC;QY zU4B9TIMPvRk*Dy1*(zenJv~Gp)t4jq`y{aVy7mry5sw;P880;Kz$Z(D@uiB%8NmK#*eMX!T|&&bMC(NtyZ65~C&z*8Zu%i)ks>3>3Xn z$)B0-E5n%B;YS54;gm-Qe1ZFB}_LFHigU!WulM5ZT z6M%Do@dsfAozU5}52o+D`)ADGzaL5=im|d6Y~_RWKbl|qd4@1=4cP>n40BTPfJ_Zjf8XiT+e`-B^L+!F zN$#ump|q{Ofc>{G6H4*yDXwUD(O}Hv-_cMdT>|TjW00v>-ApO*_#sYf4*EFYZvIgr z>O<&vi98+9g|6Ust9LSOBFrYa1Gv055b^oh!76T`0*iuwK3hjFR*o0%eF}qU5fq){ zv42E0!uo6EjKTfrr}TY1`bS*f2?j5w1yi$nRf-rD)>Ar<45?FvVF<-aJ92*#H(VYA zCP)V2B97)y{-2W$1XC0UIx@%*$6(jO;JrL0tHj^CC|4MseBCAewPZ0-U@D^81$0PQ6$!z>yIp9^hH;jvWYhKUyyUoVs(Lm%*`10=s{BO-Q?tMoWj{FVcP^*H#M(+HeQY*~gmFD6B+#W>47cyM4KEyamy~$oH#69&9Ck_Et*gQMsa(YDU%2eJ(7|+_sp|_qMLDKS*u@|K z0z)vz1$ysVnOWt*p)}o$gyX6CuEYz~qfkjDYu|$Y@6vyl>&fKQNv!}bhkY>GBdk*{ zbx0}X3Nl@K!4HG=&mP3PDwa~Js^9-nwBj5$GdMYj<6~geC6=%K>(0(wRi;NevQsM< z@$Q_8T%0s3oZ{asVajCzVSYKennOXAG)-n@05IajHk}1aXu`5zZ#mwr;64H7dC&vs1&JafOx4-H6p{P zRt;vz)6i-sC0W2JP={c?ys2K@%*aoT7W!;E3;t2zFG|^{L@NhLHs_hP`YT(4i_@!K z=|qQ}CI#Y|RAvfU^x%q6PEhupb~D%3rz;Gu;}*3W5PJ^+@K9r?2UPr;|-Vf&@F}A2}Y) zJZ7&3ZV2MD#LxKeH!`t%Bw0&keV6>!l|`%Ite>AYz@HbMl6{}+iiBXHLLrdf%xhR( zJ)FD-V8ub-$E`&liu~#A4XrD?GHDq6qWGzi+s*ptJp$>#XCmUon)=h&yN=iA8)TZ0 zq&Z*w(}gFe;ZyIVO8{)g#*0l4Hpm3p9zj3z7W@fB=pZ*B~?WMuu0d@#^P*Kbq{ zc;v(Xc0Jdg?ek$#^IONl_~CU+kPnBbXc4$0jSXziJc8!R21)$!v8-|5@ol3!nHI8Y z6w4HWm=C=~Um{hYu!)_4z6MTMobhHEPf;Ug6US&(Awt!3FfF z7W?q*!>50P2YIoW)bWa1EeJ!KVhepC{;lL2$hb$3*_XsK3hN8PZwL>gBt@coA$=9? zfyDvI5fA-9%(DB?Aui;;;N8{Pj-#;eHL-XBIKOVAF$@zxaAI9jUd&J5pxpn>xl+`~ zB=3qN@q97eV?h;h^NQf9;tBI&4sr>EV1f1tj0}jSaY2f|B*a$1*&rX9ctprF=rRp3 z|7@rLl_q)-JH?Osj}l!8^SY^B&L^@wt5uT3yP2q8IyzQVI`f*330&6kexug4K9dfm z+kbzC_&wgybMb^A<~J=nigg|^*4@e4{z8w<{?*0?c^Na$CG?OD@)_TL4gsV)e)MTlVN zw@PoLuOiBoiSM0EkrbW-LGlROF1WahJ$=GM>bWa|@Ez9yUF-<-+`0Hbgak$6?NC^1 zCs#cSF^So-nqH)P$@HE>Ln7med0-7No5e<}&TyM-A0WUN;v!WRTWW)Rp>LMHxHmXW zkT)HnW46r>pV^9n2~pnS-wz-rx4o=syUVhpf6m2Vu*_n2NaWwzQ1SZVfaYEm+awgE zS-wJfI|%5hK|%Nv;GqS0I~Jb^GMCA2pgB8yDOAn53s&xZxMryPP2?-HpUZpfATEcn zP8JS!{|)@dHo&sa+dq(LE3@+Ug(1P0|N4AGM2L625Q?9 zstJX36l1<3$1ILtwyNDVhX~YhG$5;BFs#td7~K9EBLdvSQfT5fknqNA(Omrher~hb47hRBCv%pZO5-CmEjw&tob8a;`t+KHFSD}p?B#wNugSH8TieL;ezg8uK2SdY^a~!ibTFo=(qXz-}wOi6GRN{V5_? zOi@R&uR2qGS(o*45SIL)MgTebUuDRBDIdSwOtDULHHGAD+ZnrNT_ZJ8%9NiERMKrY zqYFeTQ0u<-K__Hd+~9c>%#oDJgp_j(LuXHe!vr+*>>>3i#xjF5yOjEs3K6GgaUK&@ zKU0Q4!jNrHhbEiTKLhdc0_F}qJbJ!fQNYsFVEa~8VE`2vyKA74rUfBg+Ag_lqZF!#DJ9h)I%E)v9&76kUOrP1vj zn%D!T6pS;af)<*+U60y{Q^s6IalJo6Ys!D{RZut(%WP}TbEB@pac}VXj1`Id)G8&= zl}<$xPvw-dk9@H$&q0H8mDjbhy9OLo9K-i$S7WD5DXZAMH{rp&LWDIYLOCttbl8WE z* zB$z>h5@=hJ$RrQizkwKF+$y^?IDIp=H5qyg)FRiut3eq6T!Qqlh_UG=bi&1n6b~NJ zMCn~e3G!r;YP4tS)SVXq6S=W?KEWSToU}Rolk}OmKJ^HfN3)zogQ%>goM$uPi;IZN zxb%;`QIGo;2kUUsafnuNl|!1RtHM%+FpNelsPdY(LC_sZSZYGw{-z}X2+QI;2YPI; zz_V4*CIg}K+$m{H8H-bc&|7nFa1nn(ik?^hp>4s_ni#p6&O8v?GNhx{j_;j#%DN_dY#`qs6w0r})`K2A2a>m7Yx$Ff%)L92Z| z!XI1A3P$BoTgpmtudj*Sr;YYVX_SEzX1U$}q1=fY2#EiZd%E6h&emoj&xy5WCB*Z+ zvcD_yafEX429?^Z1M6Tte}uM7y0vhT1Rq}2boFKIm?@vM3)Ym6l*`8lx(ff8^}sAu z^pupSRiXv%GHJgMC`V~20x_J)4QShurE)S#4-YS!e^f8URNlwyZn;Mor*%SU15 zG8dn|>zVkE$c36)oz1vZRnUmgW%)a4B97@X@aKpk1<`@H+^kbdX7W29_%ZO0sZ;Fhe5$ z{DRDfS-*11LVq)6Q)tv|W5EXg{+Yod1Qd&gD75x4d8gGryb#5C7cXZAOL{DEl|x>H z-v+;8L($w3@~Z2C(SLn^hzcnO`{Q~xP+N;C51bR9SqXe(qc%xgvtgjAEkjk%ezd=?&vBMIfb)h}*A9+5 zjmE;sAuKXLNVDG(kLJ5kJQ9@@0{LF9107UhT(rH+(e9Np!wx=RikGTrM#TA{J7MVP z6M>ZTB%AGi+6QwJl_4&2eAt}K7Hl%`Py!X6zYFFP_4;e88t*_BRc0{&c$>l_j4;iM zxH#KD9Uvm0;ape!I1ByQk9~$c`LnhY_W~fcB6Dk^Hp46h8BIaoqMJ*Q+AonAVf_SM zqr%z< z@quA_?caj!kw<+|_D2z+h=jJ-F7T*5e&*i{c99z9KuUW*_(T9css1E*%oOsoYyJUW zx)zQos$5`zO<|ViL~Hv~ydPaHLMtM92Hyt1NUhz+Zu=G90!ZoPbRPAXgZe~yyF~87 zhv2lycW-{2U5nh`O&<6pFB~(60_x_u1Mx&vJMd#9@;9*yZ%@fl7Hx{}8N9EQN zbLGS5vagY_rN*1tCAV`T8`z);)bZ7;)GDr(irZHbnJ%?LeQ)iwMkt}%=Q1H_PC@W@ z4i}htA6S6K^W7>Kk`HA$-uPQ+9{CkL6}?ldgeyW%QafBCLE$z4|c}AB`alTs5sDwfzs%mJW zwjDpJhM_-CLS%1F{eA`BVcgH%X5H`j4|TCZ3{(IX6o}y3BLg>?t0OVZ{LtExV z1Vt*6;9r7Qikr7@f`xT#Z#O@sK@WZ%;+N>k(T7xd5hWHk@;o%cO$w15Yh1g+;c9n9 zR_$LT{1d2KCnek@%G$Z=#iSuio>8+~=m$fg5`WsfK_e~tr0DTG&(y?-ncmt2qyGLx zigPMzgxbqk7p@&Kw6}4?^roo$4;E{dyDMU3YYh(smuLXxO}5)jcenSm>&s`zmOvM9 zTK?;A>35*3pcioH`PguN1K5ys1RcBM=86W8cu@(}+y6oI9jEq9;|DUWiXxV=a z@&*oaPfXxunF~f(V?V6kB{=p5NeooHj9q`rs=pc`kDCK?1?lg?(lGJ#;)g>X zFfQ~G75%~Iq|5tW>r$qfu(}e4AEN%F5Vfp_*9Gje`Y=L-8IP)Yz^Q%!;nDO@jUtX}T!hSlC~VGR4wnKL}OodA=Vvw1yp#DM7+a-5~DYP37 zf$icd7$%t?_&nF{!@Y#V;_dCaS#0Tt#U5UI!(k!h`*}3TArw0TiDpTSBGk2ZR1o92 zbXoU(qaxIKC2%1(uj*V|KTd)BhBWDj`e@_Q(Ci2>*3SmbmgT{qS6J~IHDuWsOkQf3 z0k$b6RU4ClWqGCbQ;3Tv${CLu=1I|g1v}JT=EQqU@Js%NzK1AN--(5`1ggey#u17;m2s z(i;zK*;vU-Xq&Qz4R4DSr&hR5Z!4rC*dY5On6WFY1aGk{yUU5rV=+99pyg8HB8366 zxKLy|okcsn2;y-+-t%;HSFN7*ASq2I^z=@KTxBSIQj^_?pG_Ik>=vUH*5cT!8!InT zkG!hKj|H?eJZP3+T;2!TDqJVS-oG8MZ6}nJF1fU$+6`LcD7` z$%@hbut+^q8$gsK-a_mW3hux$o$OZI34ud~lzf?KP9%of0!RH|0*%Q~HQPcZLDHA+ zh>{m?YD}zRh?C;Et6StKO%oD(tldY(=;S+G-{R{{y?0@>^lo8<6161jgu*p1x4vn0 z88#uOt*+iG(dgT9XlNV-NR(CVymiOyOZ{_4$)4U=Y&}1UnY5tmah{MuK3`H|?>ma8 z9&R~uK(1c64jBio2o0}nn^#2x1teL_(S3G-@yO{@W+rJpkP%M7KhHqVF72Tz;#>4ToMe;}rsWb2wQv7cw=UiK(T`W+>m@Xiw)R?VI99d-u7 zUvcjOJD8irm+Sj3ncXiK-+p%S$A=WSqB`*owF7E1 z`x}jivMLMx{*0LD2LS zL|BWaX027* zbG};bx9U<|k~3wTuhy<~zcbtl7TZ7{pc|>DAfDN&63zQd5F&P#>(flVMNh8ncPS^F zmn5~xMy?HW!#nl>@(wJvK%U=ilqkr|h7F5uOcT@sOt?7ffezh?>X6TkJkNSdZ|FA8 z!b1&Gw>e91g8nP|C6Jd=>;>|mM>(thnMG-WvzDK(H0)cOPN2h%y_@jWCE|9X>;?09 zGcnB|Y+tVY#R;XmUUv)RWj%sCOzN$ce`cy>SNX#GjIw3dg88B~k$$gYyPfN*`9;&M z?M2J20rE$?_{+~EAe^Hu;H6sZwkQe66S}2U1@f2_-asB)qOI;fQ})28B+18x=3s06 zf#2I@z*-NzgU5N4o zUP=$wPsDiZd#XL5pjX_lzjaGHRKDW~pw~v^KWtJBVTkK;lV9`po(@g?oXxGNJh`NY zREUo-YB>?Wriqxx^F8c17ER&KU%~Dd0ra8;`>pssYJ-m3V z>gNreUWcwy(VP$adE{h}<*oF!gx_B8SuaYq7|9P+{u*{^n1H z@VSD*XR14CPliGL^n${$U?o*ck3Y|AKHrKv{|pT4X5vaUFBz#IBTX|SJP)Nc>7-)$ zNK%itpdW^iK)VF(T!7)3abp_K_Z#Sn?8$eU^K1^VJA~c};XeWh4v1nqiu;JclT;ng zaCQ1SBQJp`t-Hlql&6-yX9DO>nS8mM-IWoj2bO&Ds`tG$t0&;XLG2?BI4r zk{Aew4|v#YQ;eKS?hqYQMmc{w#(T*7tecC#g~tH)O3ipL^tMQ{&zrP-npp0hD8lkZ)3J+a()v^!dLd%OJo0f|H(yY!Pp+`bzif=L|1qirF1zyU z3UVYgGnnRoN4TsOD8hRa|3$d0)_)_sH}PMDgHGH={RNrXuquvSzJIN|z)TC33Ly*t ziS{SdTUG)J|+aP0~8Os^59imQz~@)fk0jQHH`-!C1r+$7I@< z*0MiMSWtz=bjT^;m3;B8Us=E;@mC_U1yUj?dJ2(=kw(KCx0ZiZI5zB2Wm=G@ z(N%&5rDhgDZi`D-8StEW@Cx#78~%BqF{uTaQ~dxQ+`HeU=HvM``HjrXQfoR@t5JXd z>ITS^&#c>F%xu_2jQuyDBS_Uj0ZnZL3g{@(#s3C0wGk+wqd+G-_x_n?d0z<@-C9&N zkCdo9zLE(G^I~JL;aw2GK=Woz+5PuUK|{BvOxb~gvsA-{i((h9!B(Gl!@#IllpxC0 zwI$Kn^2n-K-Ot(bqQ!ajhAq%4@84^*!g;lmEwDVUL;9lOZ|b{xj@R9)#l_3DemM$b zs0AsfpCkcohYsc(hELCN-WTVgM46-q-jkSb?fX&SMVsX0SAQfk*CNscUI#XzP7z#1R7v(rVn?;6A=mttK-}Hyl3a&h*$?_kwGVL-#>Mxmo8*B;DO>IP zFSq`=?}YKeJ%8cviCU5SQxc<|`ER~COYh35&EMk2*%8~!Q7{M>_qBHJGq?2!U#BbI zV4Dkt9gQM^k(EqA_uB0K)|LDqV#I#f8kP8v@tn4Htbfk&YP+MnSd30@SP})=D0OLh z)M2hUzOhXUvw}18UA1*?TBm`}AMc$LE zN^M!Wko-sDppnHv>ccs=cg5qI9s6ZijIQsjMF8>^CKAo%*q~Y0mz~6UB7HCmRw5r; z_dHo{G>a99Khe<)AbGQ=wr}K)4$4e`STHT>Tg-R0_Bg!56#)yjC1?_Ds#o|qB{7es zD3J{U1=Lo`fAe$HZjjpXCW^jh)IQ>FP$NngNkO)m-L$?hw!l{eT5QCEQF>Kc z30ne=G-aT$KjRA&i-IooMj?`TB-V>bp-pU!F&|}@jPCx9z>O>ZvMUb%V$OMwDU#st zKUc9r*O1N>sq$6GRa&mnA3r|Q!j<%n`1oRfv2<7A@-4&6DScbBj=UJ6Ae^C{8nA!o zLVan<#CIw-h&HJBYbeNMq;5*C)@p;cP&#|CIHzlP#A3n9LXy!MLmB-F*q-j9W>^+h(^5+=>e)Js@e|m8FgOLW}Ei~wOj`vWYC&l=n z&Op=;a4P3102>Cu9=>5DM!j!!A4>MW0Kq^$zgVcx>NvuV#)=-K+mtgwf>Skd5~xvU z2I<6^L3G=0o>9x7#Vjl{fbn?8b9{7nJQdlR=-(~MH;cq}FBREpBySRl?Qk})6=>f) z61%^gSRI{gq65h$-jgd53P?Ik5E~%NR|Xl&KuAE3QGcjBW}I{PigFer?7AwsMCYh+ zQ$y|qbR`v5rU!H2)$@sv5UckL^q5H`&BTa=Dg!JR)Zq=Jafr?yagBYU9EH^JoOSSK zk}4+--Y(=Ic|C*U6jh(97@jB%kzMKfX_9?nLYwNEL@l!I<~fRv=BHsE(9k>J6`2fc5SD@ z$$IGDEy_0wPFA3MQ{ZGJk~axX)T9e6t#8_tHA8M)D>#&<YoPnfFVxYomz~431_GrqMG18al~AYA z-SOk{?v87r_0Ydtly6oGtw8stw9rZV7SU~)s3Cw;*GF#f#4&Xo3M!Zi z1>JESe7NH}cs=y*7Ui4O!7I?cDIL5L$(z){>mYnnI(T{7H?M=2r+ual)=^-cY!p~X z#_UR~qW*+>qE3CJ)euU0OMzz7G5jbb9wi(xP(tYk_33Mz(Z4cNvyCTQpgxq;jlRJK zg&L6)Z=ip_h_Vnc&rMS@#n#%JQk@)pIplx zO1xnMgSK{*@8mhkOU26z-O)|7nw0726D*L>L!E6VUux*D|BU;@d#8G z!*xUjG?(OkVsy1bTRU6oN&-n9GfyH^0Bk^-l=qN%9trEF!j=4{B0GvEac_M*rjDX+ z05z;F_-arfOmqDOUKv1%SEevtz}5HQF5%5*`po&@1BI zHX^4;HO-~iI^Ki}5*ja5(^F{XAGN}s!BM`HQ%DBt!f8JG>Ep+DA7rDzm{zmhJ)Zi> zaN_&vk9N#QW0uYNt<%IHYM+pHwg8DsxeBj&$J`IZnS@l_LJAkTkXcDh`ozI)Qb#LlBbKH*>=sfAJRkjNFdnDogA%ilst zt*$+uD;`0r;oE4Pt=TzXk{hl@*vC#I;K8f-S+n}4Z>g{&9jl(XX?7_IkBqin(FhM7qAKChB>g6JN+{f{tPIKBjB$ z3xrWP=`(iuYp9p!oPI}7T%c>rQApwfK8Y!zopjpS1o&#c&ae+s^ z!ucQHXpD6Vo7&&6MDiy0_v;{hQ~UenY2W<*etFtw z?eE+Aq_?e4dX3Q=T_m@|7f9_W2J=%+o{6}I9aaL1P#+EO6$Dxhl4gWx z|DU1b{{lk#OpM}(xU%7o3HG_#xn-!=rhKkV`6C$CYO)^XemL)4)7|2UwM3^E zfN)mVZ8YBagZ06DaABrRvUiJs#HgtRHq^&nZ8-LNCxM-h$0R90@`?+h@R|$C{?03( zOg>Tx8wGQihg5_fWquqpNafQF$__j%2p1|4lm!qB3W>8+JfH~iU_c$ccOMg99GF(? zQL6{vnNQZUH}@Nr%2)IiENf~KeASyI_`UMA{}b)gQ8jl=(0Dj=q@pUg7xVHl!!;1=<_o&~=zg-2JH0=w_;6Ybwmw z9cyyv1I9KSFuqj5Cf3oMSxIw*OotEG)bfix`Ka7HD=Q0(F+4w*sE7+?U1Gjv`wT*= zPAt{es6RZL8qOQF%o{iexQ^;=Bx8|Hnz!dUnyr?7T+aOGIp)fG-ND?!E_u$vV1#|8 z=vZjh)(IyFPXyy52t%1pxnTq+n=aP~yl5ec7l;9N1Ph7VKxH!pHi6-E;tyS1!+Eo}gAg8^Kp z`9KF^^z#NSPK_?J|_#s{9j`;{{ zn^2^FIZK4Gn|Gpf@M!Yu1933d7tS9I2iFY%??1i&c2d0Uw9eZ4ZV{!$o87G!EntX?zZ9aYC0I&!~2?!_!^UC2OL1+i87ZU9t+L52j02 zBl1CY$y!K#FkP|&wHweSD^NRAmuOF!)=@&29L2h%FwruE&Lr~znff3)aHQ^0pCB9# zA(fyp3o5&!ZmWhkA?k$eQ{Q7l6fiD|(o)I?4%*Fj>!5jf(CQpyNvmJg+x4c(LYdC> zY}{c?SqB&MQJ>+^)rW%GO@_Eh54+Z()PDqr9wlL7OQ)T5THOTey7Q^wqeu-OKS?!w z|5<~!lj3cs^`X`9z2pMgDk2|N4c~9hob91@ref2cE3L!C>U=og>fDW1=c6aNI^Tar zo$aJ}+i891)%jlX-QTu{hs< zqM7ZX_`xM~tyyqVlU_S#wc3eE-=1&MccMvu_|zu7j@_)wzOQW_=~`Xdkr$49$8;D% zFrHvh&27RqIjsMIz>9`UL0YjSmSC(wp$UL!sG8kc5@4!#KRHeV&eP6 zGj8nQ#qMK4uL%11^M`1Hr2^5S2{OdgC$5?YSItI86`7A_opuRmCH&=D^ZCnl;xnLj z?xbsGC*3>1bLBjX=UICV$T+DW^st=TLwaglc1KZly2Jz|HArTkafT|3-$RmOy$uf?3KpZ-_RcRRDS#UpxrBVNVNg$?k!bubr zV$CEKbBz)X<&vP7iN>l7pBzj)XhnS;47Dl@B4>JIj&K~ise*YqQ_o{0$T;v+4kIj( z6B0Z#!4PQ{*+4?<>;c_kZ{hHVr|(!wx$R+f@GyYG4r&BV!p`f;InHJ2*lOW z`k&7cd?m5|YYWv;!B=A6dYz~vzoKO=5gZVG_99t7t3F_%1AcAs>*heG`na;ex9A_f+flD1&y_G~J+d&TY}MyO5EfkPyH+GQTGg1 zq=u&l<~3|+Mv5JU4pKXCU33umMfe4LGO@QDk#Kz~vH+*R}4@~danEc2@0 zY6x0;brXbyL$-{wPhZkse6P4FJGJLiZ~xnws)z+@bUdC2-1kUSU<_ENcr}vina*PZ z9{I#~*u)nkByJ61JxU|W1q*L4%%L0j+xJOS_1-0dgk$Oxo#w?%UfKFJ3om6`5R#^v z3z_Hjv2%H0mI(cP;-SpUFnRJI|A|lJJVh_;HMCOsKnVKuLZ{3#>2OpkzVDSXf31|g z9F6DzP=2n#M=B5rK*s^3xai3n2j4kydv!4)D&DWIz(Jix?d>hji$Yo=P9B|J$)@Fv5&`)1HqFEKWCHBAtugAb=6wyzg|>B z)$w`^=c*SGWhvL+ujm)M3XQd%MGvE|=%VUiAv|I=S?9N0ka5+7r&$?LB=mXDJi;4l z#g$)qq1IlLjY2+xUZ^ZM@F?g5Ij#mcNh~c$h#jG(x~1h_yyrDNG2Y;S*8pKC9LOcH zDoNB7vhZ*ik|7r4Bhy0**Cn0j=GGKasc+>ex;){@yGa5pxxy|EMim0R(;qy5Ogx+jE$n=1e1L@ZJst~#(&bG&qDIn@Y3K4kgwNp z4`pteL@(S75kYZ{P%Yqze0Ij-0q`xHkec;8&v`+`^?`NFrQCrGTJM;fc$C}n5U+oB z`YW~M!V2&eT@0|tK~hqRNh>4yj+S;%d@dafNGSR4Qiv+7R|$1s zm*t~t$RFo(`0c-sos*YdO_Q21i7gRaNyv zbuNMyMTy;v*BJHD`~P|s=W4UZTM_fYBxqZah&a*P{NDy1ulU^n#%ca?IPL-;Yu|rg+lAl1>~LKMaQX4=@Or9p;U0EiJI1Fa zY)q#fti>4|6SHMscH*+bi~BI?lXG|cu=1)uI*X`#mD@RKxEJ}I^V{o{q^Tc;echXh zCurbBdvmVzC6^lrdC2; zos&~huKz$7MZ4!))Jk%h$n`>!Su1r)kwk7kHFB^aN58Z1vVI=2 ziLd+jUp~9`(QD2patvY1t`nFt9|a-3lGEr%Yh|reewQ2A9~m2Z7F7*|Ap2x~s!ek& z(!lPRf(`a}#i&SXpW7lBFMg0#6rG#5ka?|A`8{PHiB4XOp}yJZH2+c4LKl+UX9MPH z;~59*x9`G!UQIQyf>U1*@HbF%3exg`=^AX}dbks7--SJi2LuZceFk}a<=h-&PtE}; z1Zt$^)=!;7sWQ~YMyKrrYRUOY&p>61H3mX`H1?BX$vKKC19=}$g!&_v5c@nYI!S;d z%Bb&5+n|+Q8Fs?TGNEotsr&fz2S{E@rI>O~^?D%JGCc}|3m~bK{fif9VZb^L97pQg(vLLIIWJh6>)Ov=#4SDM!qfE1FpF^~Ay8 zRiPkUR8qaBCmaZz#~@{%>`Y8cqta=h-;$I2&D6=A9L=P)uNKKN4bcEoPY&OhxhnOM zVA@`zl@E+K*TSl%g=w~bYvD0rxnKNc$$rKq6ZL2Y zkoO$H@NqV8Q);QTbRDl$Ufbsu_F^Ze^QS+EfRm#GIa$r-EZ2yY`{cfa@7P|h1VNm6 z6O;GQKzqGtifG!vlo5mrVgn7GHa^0RiB(lfo-N;r#o>yo1t;2IT8bQ$-dxIDwvgHK zD%-YXyeeHKdHgdSQ(>6?eVlPWUP27bjmMf+iW7P2_24NN$(S%3@FsQ7~v~z^aLr9+Y(zFfZ9?< z&^RBtLN_nT74aBUlCUs|LON2?Oz7jwDD+Wh7gtM6_IBFYL^l-@^oPW`7&$DsRV%=! zIbEn`oxV(acVFp25NmyIS3~^PcZiGj6f&0*rbd4sCK?z=0n~mPVdwJ8p8Wql8Zgf# zVRBOv`LrSI-$qGUB_K8X$^i|zK%XS279%n?c#4H|NPTc42^g3A`rEmCnK~^#=zFZM zj+p>TCu__dxX}RHC(NKg>ezSddJ9^QV_O?zci*Jv^MO}-hR4iNbLWRi2z-4V?;})( zRlU2YyCe`Jxg&sR-pFq&<;bOv)8)R3`a@nb`D!^Kl}a6b%E8Jj&4^^1aZ}cD3T2l2 zfICsiN!b+%uPC`bfEM)3&!t53boybSe&{;W{KrX`)dah@79MC z>Jol1R7$PR1rDY19J}A?$oP6HdPmLLm_wT+%?c?4vWx}F7`g7BIK7vNQt85)RjGU* zDYshUv{lc1KJ$Q58;|4+kG3^6UAieG$}i4%SVz#>xJL;*)Dn?vy^LyjeUSI)SPI{* z_U(h84kO zY8{Y!VzXtG;3Z^Z8Z8mg!G1Aa@j6pOX2X&~7j{SG24Pxj`6FTojB1Ih5?uqTEAccc zA7P8nS!^BxJ8kiic@Pm{4#eM5?^jOvPsAJlrai5|Xj|MyGELx=cG70vHxNc5I|75o z!L2G!Iuz`Jcm7F2*~e>N&ZSD_0~BPr$aN*VkRzeCbm>hwteeH(i&8ve9qIyB3(bA= zadUQ{t^IDJ@)NrzSH|$lJXd*A1tA&Gn>ZD8pP)+;#2N9gu`i;UXzC3mS?X9{Nu1mt zntdXy2B=TuO$Fu*N&D75gxT&@$^k5SGLhr5XWSL@hF}pNV&6Wtsn%E9~+N+eHXU}p7diKnAuE?8DvdL%9ayjzsnVC@8p;hCktY2{8WhUHNrShM! zuU6w7goIDVMBCtXk!5bXhM)`E@h3J=nVi#B8gb}^F^vJth3QCry=Sl%ov4h00zV)* z_IEV2kLZYARTj2)a@(&QG)TxP7sPkSZ_JyFBOA`agvZ;vLg|Oq<2@0W`WMLYFdb_P z!&#+*5JJ{mmVcVi7BD-Y;OgN2j=nGS=Ov~8iRuCR`(2d>#HpU$RfE(+?+JWL)o`nx zRTTl%Z{PlIKB~U1X6o6qXXqnC?$e{T{9&bLv{S211`)t;x&vj3H6rV zs<%BLA^K$5wBLV!{-P-P4?uW$MY8%posQJ;M2vj+|DOk8A3F#8l}fvTe)a9hp`w{@ zbiJHOI2b3cK_~<2jU=)BeVvO6hI3+{jge)32h%Vd9K&*E$(ybvPoW{OM{;zZu zYg8(o270fWbtWOWmR&%60f@_cGKi{r2vfF2Edsy&okLZCi9U5P(?LFWeub$gb@FsR z%@*mpxD5#UgEJr1gXe$$bXR3T^$b-NBdRr24X_x=D+iM314DH6+n2u=O^yeUaKY|D z$j_m;uBuZHTBfE9y^7@$FB3kRqPP+s8E~S7FA3t&VY(iDvqZ4Ll(xolOM52aBW7x<$?oV&heC9?z&l0p*bJD&gXNtDS*Ig!+Tcb||PDOhC$$WAq_tBz$ zO}MDHa8xBGz})Pcc~mO}bBW8j)P!bR!t>mUw1iu=zs)RB!G3it1a?MhZkVFHppmr_ zdH0)|oO0uAqL%Zvnkve8+E!E25qW}TDz7}?54rzRbI0d5#fHs zqEH}iK5uON;8=bmh|lR2(bechv>yY_5XTV_%xNORRu8X(169!$%GVDfq14B>p+YLy zfB~ZmOCoq8LRNyJPjY}(vK&(IiaY3@w$=GXaf=WNXo7Mko_BC` zeAL}H-Ys-Es(a*$c*vOiWuITDk%wo#ljxYZ$d0VDQ-eSpCOpxZ9*x(iRO;w0b4lG8 zPix?@QwL|ly7neKle^z!<#jMeGVChFYquGnD@XC+uG-}iPy0+60f^hH)Xh;Id@DNF zi+0yw*4O8W^-y`_U_0(mc?oDY3Uxy25(kHR5#W)Z9{5y%fzrg$fgHvMh&fw7dUK-p6UjqmvvbkA z8#PyAGYs|Zv9lm1Lykuv)<0S&(oGqxXzhq>^M~d41#aka*@*mJB7b6Wt@dZolXzc& z)W(8Gwd1cr9eMN;8one%(FG%GmI#ewVp{@IFiJTC=esg!%@#oj{f5b#@b0%4`i%KE z9$V&(2T6RI!zABF|19wabiFI`uU%d`7Y&`%Dq_a%Z*NUu;-D4Nw^+q z5>?1Oj!25*Mos9BqS?Y9IG{R-#~He6RW7OTo@Go5l`#=ma#?urD?% z23p(jxq?s~CE%)jK=%11<(CRAI~X9aeBlheWxm>jN|wA`;Opj{U*EqzLkE1)kAjf+ z!Jyk}cioeAy?xYf*AJa;vwq^XyY}AeS>qlBaXsT`P47~*NM@wG`&KSnhoV#6^+3N^n9w9LjkYG7K(&R&Ywg(y#WsmAD&()Ws(b(pFLBlMV>v2 zUF;iRUeqc~xoyWpXkQ@MiRz2rQ{O#TbX{nF$;cP&q6w5V4JAn%BXXE0_oU!J)TmTm z0@%|b9|=!VvoXJtF9uNTTH&aM{O3ZAVjplq6uh563kNZ~lnY4yy}D}kBB4j~6KW;x zvxvA7B#y0NnYvt=<;d#A8Twk+%ToNPYz|s$=qE2vV4=EdVJ{frmbGk0^Cp?sLO$>v zYBU=9x9q5$q123lC-a_5jAo)`;7U$k)O|%@G&0i`>JkJRgs(o+JO3bOohDNiQsm;_ve1dq|D+>?kkSYq^m3x(E9W+bkEE zqLVxG*bH{Iki&7Y?R;NuC_dT|1RCCQ<#T5_1-WEEeVq^s0(|3PiJf$nQJl}?x2#Q{ zuspm;hk)d`I7j?ai5(AYBmBXB41#NBM#d;wI~xid4vj$ zMmKTUV?()?YphP!-USkj*aLNZbZQ4c&ukDYeS!KeH24?~AoJbCf~Um{#H|QUHlgI~+|=J#UTZ&9(SN zIVAK^fyJuxcdwC7>XNUXBw-*y5XMZrF;Xx)j_}p_l)n28{R$4EI@g-@k*ijt-Tg^$$$S*+@yYW{3me;buF)yiySxM!!p$epDoB|4A;_Ph`8R4O!_%-VW^_8@Xt+odA^ zzvw_J>g2whN5)?05fdu>lYM9c6Qv)rOX7nA9LdXp$S3NEQJ{g2GJ`f0BGBnqJx`rG zv+xq_T_hbQFKVeSX#c&2j!PHSg%{ps`9vc_N%=C(mNtp_l=#vrl2C?*8sUB6T zWqWNkBqh^UE78!a1c;^zRAnJdmGn9mWyV-#A&cU85B5_zS|cC9!oi4g=c%l`iP4EX zq~c+Z#wW6s!8QrzQ^J`&zn2OSL&1Q=10MwTT?nL1X^MfW{Q<_oe@uebQRAr5R3#aj zjh4E0+GsX9$`9c!6VxGaY_7dd&MM}RzEVdQ`hdM*MQ(C}LuVAFc9cR8KP9l9Dw)TW z*P3}!?c_+9^IQ~Mh&;-LDo+uy&S%$+O69*Yc&k!Zejm31tmqdScS1T!8drb??!2Uc z=d~Zt&y)*^X|m~+l94vqtf3dMChJKH0QVvb1an@~E+8t)lJX~!y#B>;Fpqrpgp~k& z0P_rH3#8!Y%FQCqoMM9etux|q;t|M(LK?aC9xh@T#pp67#hhasqfRXfXpmDDLzXBI zY=%TghePG<8f#Nu=OPhwOd9A@r+wHunu>ePzuK_|Uic3F(Alo*C+(Aylh$$jXuq^7 z-^)+V=3+oS!VeO84G-Y|jXyZ^m-xn1`){jxe01DO+JD=vW~*J=e;?xKu7Xgt*Q0(g z5xjbaK2<6ReW|IdCeKjvdK~#JZ`{F=QB|(j@R&NizB7K%sM?=UPrUZ4XQ)++ug2=4 zdWNd?*!WPdSFOhsUa6j`1ivyY)Bg&pA3pZUUg3PsV#%BkzIW^|Ubd z;@p@$1vq*h=Asu8k72jRCvbQ?n0P&i{}p%~^f+;t@A5mdasBJ_&((@zkTX7`ImjjY_XSqF~M&=WD0do8#5TI(aX}zfrPEA~Hy-{Mxs_A9n(j~PF_4Qd_BARz z@&j!5GxQKQZ&E=kWKXDUo~N#aI2e6o%o8*)9jsE5 zazyf~+d1rXPmj8XC!IFj^{9Ut(_Rg*Pdpw8ubItN2RkG3k&X$Q2fkpS5I_~YpB&et)auFeB>nOnQpU&TF31g zlC3oxr^ow-ztKp|A*IzpG>R;U$GtkllOc&#J-9T+H>RV>X*s68xzS9`#T+>hqmb|s z^ITO3X4(^PaCk|8g>qh`SEK4Mr0zS$lT?nXXQ*Q}CP_H5{=P|3c`N_UG)?75x0~_b zz98Wh_R`ZMIm0xYf`IzNk80w#@~$T*MNGUU;&FN1}r#1Rj$ql&ihQX>Qml7XrQq{yWALW-*%nKM711blkT1{X4 zf%)PC{S%t(HvgA(A7bBA@4&UfSD^cEa4?1Ka?d2+lk_DSBoyJ*w+H8`8!UQZ3h#J; z!%L6)DOnm&&r2$UPOFAmt&C3NkKY4vw*}?I78JNUT~Jii>+?Bf#H=ne#RMLFtCuT$ zHB`^)DOxJ(97C(T>W%d?|x}Ki!?eq zK-)y=@;KvRu*7M%hE7lAe~s>RT=792m!S7p5VYqrzkc|&dvk+A!UN`WVjW+UaO3I0 z4VO4{j6vHspO&?ulZGBAbDA4U=z&V;Q9_T|&;wr$XB`RX!~6#|Dt!CY0>tqQ9W@`3 zAtBEx;Bjko!36e7onGp6RHW0}tvPjir_|}CPA_%(Y@H4%&zAJpTSUtjYj5=YIclzQ z0+8hFX7%>Zn48j{=m$(aTbLfT?k?PoV~=9~!syw%c<%Obn9(8NuKF|ahmd`{)l6I` zd5KA{k#{z`VHP@^5EG+M$Sp69;S$q4y~(BTNis@-e` zH`SD?JBt72Z^?DKOCA{#07VvI*rP-LoPW%B=#@j6xu}yC<|zFUb;+x7AZ`op{gZ?& z+uj)8qD}(wa25h5O?OhaUJ~wPn^U+nOP9)C zT(maJ6`p5CgZkuJQ`Vk_Mds1Y`alT}v%nVW1c%tA6WMMzc|9vfA@N-j63_?(k6DXP zNCJX0`d?jr30Z=bzixFDzoV9G6Frpb7%cmFA`ao z)A>tuw03^_k|F;V-%Q(R#q&KR{|Ztw9WugYh>$nf@O#WjX0j?plzCifqRizi7%$$# z#dLnZ87l3w{5%P{&5k)BgPds!*MBBg5!DsD(HWR8q4iN6Q7$M`DfuLKzB@T>4w~(1 zE&bdWbD?z7cl|-D-)v?diNi38K@O zG6Nqs3o2zFOF)UH1In!fhuh?)kPii?xmnooaLary=Et@7a%v+RXf+q}=U584M#Tbp zV=`vpZEwQykeo^J@3~U`cL35ZkSUGCwjjUL;jN%FBFNTe)_XeRFJ zar!6e@yYmsU1aVr%w`=p>pgJ@y-FmTSgUccK%todtfM)HP=TS{8pA2J-}MdPhvdq& zz~Ftq4KVnvA`!hl_yr27^GSHWcHQQ3fcJ^mbvI-$Ev>qXK;Q&)Dy_P6Veh4^y0fr% z+Rr@)d&9SmAKIu}BJB8LiLh5e*!XHVZ)kf+sciuS>ue(gYo${gO|^ffOrOW6j~kb% z^)kIK#lS0drvzTx1-#JS46|4&wNNlI*?W61l$$V;Ed#(F9Ar7c;?|oc zaDdz`$DHImwiocsEX;{zRt%UG&ul$4HKsX!t~jn4*;5|riD)I4|L_0)@Bgm9c~gJ+ z68-ek*?4@G?qZ&?RS&rkB!uy!dAtci!bx})e=BQ9;n=4_&6@1Z0+R%?7jiB_>JO9O z1&^I**_pXcMFXsZ2+9%88)1k_N8hF6cJna~+$$!}44yw3_eq#Pcr4%x4BaazX;cne z@xXmQZQ$;Ec;0oJ=w~d5@7zA+F>0m(d}fPSFnBN4J9w^eb?M)Q2Jy5U#LOUO$Li&F zjMZs5R$gvNh-T@nA07&De7}%JD~Bv+$UY`bB6CXAh`KKE*8|~y`SoKt zGUi4>@}%^rX+isOy=!4ooGPh)&TsA>Q7+gp#N*z-CfFC$BYVw;8inujwT zg7L`4^I;d(AD26~O$86XQ=SpjBj~*zA0*K^hf3@pL+tYb2_d#Poyu17BFl-nPF2+_ z6*T=sEPuwCI13JmWDlg-2U*%+@wZ3-jEKTFv%h%H`-}@zb??JSoesIWzZB zS#~41;GSmsrZOs|Eyx0U!;c?-F5Vkf9g%1Ja?&TEPXyr-3xFQa z0)j=gyHe~U#>GCWBdEEz_Np>Dnp}QxN(z$UCFyxYaN;}RZ6JCM35DW`s;8c>{R`a8 zrb2?0>VH8(K?l^qf}mIOd$&ajUg7~@MGp^$AsOOyX}lb@UeVu{B+IEN?|0QZRql+) z82?5>t_#K=smoGpsP$ThoJKY8p-t_a7qu+NIPkEb{xDKjRgZFE%}HB@MLx8-aN=DY zx0a76tOxRzAA{y?^&Yudy~F~e4@q!gWeU%M9^jv+*erV!UA2q<|7$`*@B$C=Xtx;O zN?aY1o7Bo^D@sF1TBj#mpFER#zS?+Zw=;vs=V{FuG52r19!_K|wvon6oR-gh>O0=V zCFi**yv@kGHLHf**m&Xzn&aDL#>>xp$r`TQL4r^AltbUlq}?OTM>UkUli6Q#h;xFM z1s=jU^8amUtS0YfIwP;;KddK^>aBHS!pTR~P%ct@q_fJdB`SBNi%hQcj`m*X*j=$V zc4GPgB?;H;a_(*Y+*@weZD_vDYu-uD%YeD%tei&RcF)zUr|is$nsp#u(=3gWE3Q06 zXIV18#rFOc=WcsD<}UpErJ^T}3V4VRc~s-UzcMC6O`Vlv|l{etiA*|3L>R+5jfk{OSbki&h9! z^(x+?RlfaYJwGA#oe>G^W9C!ALh26>2Gk?`K+8lOzJH*_u|C4S>rsDL4@d~xbKfD2 zKRENJeWTfIHjfSu;ooMnnfmvrdE7nzOY88sbKGsW+nw%Tn(gl4VdpQXxu!l&{h4rq z!@o3F{#LqkJMyC?yVs+BFcH@F)`(RPmSP%`oO{^!nZQQkqCQAUAUi%qa zO;WYh>*&W253r9>y`EH*#Ltb@jTK+^yvJdFveHzuhT5%G4Yj-78cLUeFv{&b?jHIN zbncNb9b^#i)cL5-aOmdP=-^+i!@*7;(!j*enDsX|{bY6M69M+6)1%g*xfbGJ^f3zA z(=1eH5wQ)mN(Ipdt25Nw80rp8^Jx>N#TQ7`U zpBLGr*!PhM?1AYJ34BR>G+-fu1q;s0CIJ^A!D9sBEFnjNAst+cFw#pJJjsdg!Xr-F)F~lxapwI4J-a%CMC@LqZeD*jhRhKVg-g1fmL0Lqz%!KK1YSWw(`W}V1 zXOFxK)5#YFxfg}G7ll$5N2l9imZ`K4c6V}u)Kg*VcJDJ$>8Tob7HGXouSq{gqLNEV zaWH!(oV}gClQ(pQA?-7Q+NXrI7X`LY4{a|vRY@KB{B`5=#2jPzma{=U><rRV#_x<{Kclt;t-sx!}_s$dCi+)e+GmFk5i`-Q7o%dvzx34eCJz#Hj zl%6&U*X$usIut)I6G3sdfn1wWCgTA0U3x{`2`(_GoQYscE`qrE ze6!=Wg6|Pg(3=Ou7l`>Ngxu30*&`BxX*3Ag7_Ox7_~uY{>e7lZ`>=%B`g#eocNk`C zci;ztJB3L)XMmyU4nI2z(;t@gj}E7AFZ*^4sQ+p-bfmmXj9y~&T$)RazKs}t)7I*f zTm}P-9+>+B+;pC8oJwOp`4rzj1s};+oyXH7VQU4=h9kJ0rCuu|60S-5R2TqcL$Q z_G4pX;Z|njHpdmf(n$%HO0X2Oa+|?Yf!!x#udJbEz|^+bex44VdNXZ(E#XyZGAf}} zs`;IQR_`Lch1dWc-Y8ef?#)dk=p5lO^M?p~UVN>wO;)HVSN`J2s+FjBSVF7yLMwzh z`jdpLgjj2ZSP5$sS~fXbY=A1jN(!(EEGGKar^9;pOXHbDQGlQJ_t61*IAZiSOPH1i z)1v%QP{R%IbP3KL2At`236i&L72GLd+M|JKMHVnIBCrCpfxSah7VxbzfUShn+9giA zH=I`DvrXc&BAb?#q%)0%N+>jsR})yM9Om=Mn7AbxDos@-21+%*-56-ocBLpe>@s*| zj%eHxuatO2-`)cMrU!Lvs>M%47ADKUR4Y8EL;YL8RW<$I!}&p>yE{Yc&OxG-b*S`> ziuzUhMwO^2-F%6P)TIJ5&I+*T^t41pB`S(pxy`63!bEVEvVe1vsjz|;fI$(?+5hbC zBg=YpfD&{pN(R8OX?`3Eq4k%~I=I-qGroAXm00a?i2 z^eU6f0*~w*AU(e#As$*;10)kmK>9F%)Fn>;R*({tmY6hgXIC(3)>7ALFAqtNW<%04 zpND#&M3w7Bm9x>DxnvIad!8y6Zyypa0q^#g0G2aJSq*D$E5gJIsmz%eV_WdmuXgSw z>JX#{tAtkN>1T%KCyS{n)+g*zS#b*cOtq|b+uX?1%SuN3(@GZ>RQ*VfLXvIs zKxA;~8&IDbkG{p~V&oUCC>pXU5sl7&B#MLAMsIa1|6l65+gB|xkqAn9t$IfX!GeKq-+qvU6jU$u*ZB+G&% z3xn9DLGlJHuRus%i4c>*;wO_i_g!3l!@Q)+g&h9)L;vy@{?AAQC5nbrGW8z{>woAQ z2aY%4f`sPpy8gTFGneq0r9a?jdi{^XqvLKfRsZAguz6J0|9FTWDeAd(Jf_s{%&p)t zXZ?<%>K*y^?S(3L%v-mkxN1kbbW%=Pj<_g?0k7$j9H4oXDX9(Ct5mU#bg7MKu}xX; zVtViH)b`dP-@Zw@zv~>I9=qL4QP5l?Uy+Hgz`!?G9d=4Q_XU)ZOA_jm0iJjwgUOu< z7od*rzUA(ySi&@u>6SJoHWL@}Oo_#H;E%1ZU8m~8=aqqPW{8+(I@F^b{h4Dm%(od9 zTMYBf0GjhRcHf38y3BcH3^KK-rNlWj(5&73Ei)gx8rsL(2wbE0T1KxxpO zIaSOlcAanUDzbJJ+PVrYT`^>O@$OgjtDqkJGg+@Y*SJ+=+R7_NIm@gS%TK;ZE61Q! zXwE7!X62i*@(o!zW~`XRtO+Y=zmJRDC-+jbV;l?ZL`7C2z0g%#sof?lbk$~R)0Sf$ zD%!bEvu739sgE%eKK35R-8asl1!8Mwo5xgg`7uGq)I-oGYfLV6xhYq?9U3~xY^ zTUkk-jdTiBPi{}U%l5Rp=AP!PS@}Kve#xG;mhNh6-Fzys4{ov<9emSVFxL>dj~CilD)mj9GZwdqc+OSD1RZ?~xw39ZR%e z@B!n!V06o=gFW-YASC+16fywtKo7rAE^Fz=q1a@$VJF*gZktcQM)n4}Uz*9#ia?_@ zla(l>L?P=ra64WCgmg;~Qi72AK*(MM6u`Sh86jd|ap#gN>X5(w`*Ss9ObqzfpK$29 zlwbY^nGqWQ_+0&FqX~}<0ZGV5>~>Tl1=-GCx`nTw&g=jDa@P3A{^#nqDM-d5Pz4gM zPN3D!k_0}5qr;b!UzVT6thac{ro1c81mI894J(F6Jg& zoRWJpopdXa$uOD6&@33qB!=;8sRV{o(-+#w3lpge6Nw9}Eh}}RoklT{Q_)UXXy*~M zlNCm(3X?rGDFF5Ci|VtfYVZgKV|;EZO#8zS~J#f%_!6QmBx(6kk)U8xd0}} zoLk$i3&SZh_$3G~qi~+YR<-t&uz>aAJtpQWGx3#}rNk`y=e{%XEo(_Px%bv{im|3- zdnN9+rp(h{+IhIRbx5m3gii|*ZbuwYnF+6iTDy`7Z!Nj+9( z%yPFLY*3n#O5jiehv@Gzc^xWo!&6UQw|(jA9yOFHGt(srE`iKb1TrOFDf7yecxB7+ z%BJ$lEg#8&-E~LMJZ$Q?WqS_E;bM_0rVEZl4E@1uX9y$xn~%M!F|RISQQwK^)BF+TT}KxdU9x{r|O^^Z`FIc*s) zr-A{x=cH~+4A1>$c3a>0q|&?1;kUJ#2;EPfuvO&ezOuTlUxswru!1pKAeY-LDcYJr zy2p%e_aH*k$7g{&ZnMT`oAl5$rK8&i~eG9yQDSZx8VU5?*x)cR~uQ80cLXZ6$K)^k!;> zN+&T(D97Z+J(b}jX924vpU}ctucJ5ghPbF+w=#LyX*>#d|7eN@z}$?$N3)i>3;w?; zci_ZB-wFuI+w>!|@$aVkd7U%{pd14LLaE5q^ap`eHk8P2X0KYUMzfJ{InLc6qD(fH z8AEB>(gT{S+N|gFum(BGIj3j(C42V-D^X9aMBo#6dq-{p77F4*vMty^KmT0)TlB~0 zYEAzB{NMVb6}ymV%q5(L#9hA8!rAd9!SzTh$?qT~Wjr}BnweZ+kpcaWP{b^(nc>}P zlGVx+t@O2-qSbzaqP4m-WIvYP>AzIpKp+WI6E*D)-|mxNkBLt{6Zguiw@f^$dBrB8 z#|Axq>pStyU>pPExXCw@JI0!sS+I;vAXJh8l!TmApjc)#t za4R6bOa0+JG3%Xb{gLqM8dGu4;`lqJqCg&Jq45%juM3BtC3H^J^&Gc@ ztm8p83H|ipn``W54O}`Kz40%I*q`1hL>>g-snAn?H&gIp4Q0nSJ${zoLA#c}by1(q zN9LR78ucA-;*xXl$xV7DWw^R3L-XY2-${Y_Rzvcyb+B3g3Qkqu^klaEIf^_w(moyb z%f>}wB6KuDEP6l4ZleQ~IQZ3r0W7I2mQ)|TKg3h}S;+sxC4t9o$HW&4d4kOF|L7jJ zGxh&k-Q&{#;~{>g_@5?EFzu0 zN?=+7(-N5O0{co}S_0DzaIOTVyADiCkAay7-06|7q0CV4{8@J95G&_YJLbdFJKvu3 z{8vwmYq9i7O9+?Vp82Zk?hrdwcMxVw2XmYAtEAoVEIt z>rQFs-%_C|*PSQ3?&O$p9`Wk4MH`M*XWBA!_!O6+kSMZv&tVgPh2YuTeMh==z34q` zPO|rgh!O-8aPNUOAJfIJTZ-Qr1mwv=fi*2-PsMqnh;cp>W&VT}_&27U-(TqV8`b)w zWnP&jU={XF{7au*`}f4XUsdamm3b@-M%cf%x%Z21{;{&}8dI^gSs2P4z?PT++pTFk zfTMB;Q0@TM#m907uq5Nk9YDDQD0cw6$-Z(2Q0@TA9e~A}`E~&1fn0`-4|5>bKAN^4 zIDU%z0TrLHumH_S0JCE2OP;)GpWAZVeLqq*_rcqAF*_;6Yy~lUYTLzC8TV9!Zr8;> z34&gXh;#YcUj#P6E9i9^yw6(4%N2Y(45yvLX)E|?xq_D~_=7R7T)`jB3SI^jPYEcV z;i+9_BVQpK`4V36k1jEJG9on*!>!r#^g_4S^0Qd{_b_CWVBsL}8S&qThsVuS{CB%^ zSmytJke_Ms-#^OHE`z|g#{^skfv52$EJ*9E37^|^Ss{M64GLu-`2C4pS*LOxTzqIB ztXV5ix%O;@F|!N?FV~)X=i)A{J*EFihKXfB_=5=1UXc`;zreR^sr^U2aR1>mUmN^Z z{>rp%&dQCKJ}*1G!6}2y?@cUbE69&bVDE9kiZw1;a~+!cx!7#h8dR=6+p)zd1KrBi zr(AtdxyjjrO-|`#Um#Q_g5_RPU*&#cGe<6E`1_XZNlHKbhZ2(~Z2?`^y7tr@V{^Gz zhQqIciDgXu${Z{M<5%Wk8Q{G-56cV#t8#FMtoflZGo z)Og5$&deyMjO3Xqy6NgfVaTR z_0*$(<29(K*PtGD1id1?YZhLTkl+2m`V#210{jR7))JMN^@&t^=Qf9VUpr*#wJ(0NH5DT)GBUvGFnD0=J z^hg-8koN==*qc|vK6S@m_V-c$7VR0qsiALn=Clcp6TNZao2*I-)r9|;b3*lCy%l6bm)F%+$DM94F zCS*c-E(yeFtzdFxVUpPKe@o)fOG#V_9ycy8o%S?&c>*z$BgzFEhIrij*97~5dStKJ z=r(I8Qf0A3X}Ow&paptdDg~GK`V;4ph?Vr2e4vIrvOkYjd)IBEyF2;H!UFg~k&Ff` zM6U%IbM!&rTe+80V!78R0~Qk0C)6LJko;>xxFGI6de?0}83niPGJ81Hgz^L;4RdAo zbh~BtbOSPbTFUI!$m~=r%@c^o%a_^H?Uvco4an?iDYIK5v)wXwFC&`Rbr&_FwhQEN zJ_C*tzg3Pdrgi@=^tcZYcb{xFLebGAfaSGVQN7TD8LU~M9{M_{KKa#~No-)0d@&bA2b z%7&ad1opHPSc{0Q5!lnIR-D#Ta>|M2G9*II!Zmo}Y{-)HUUgt0Yfhz%Mw-Jf%bEa> zF8s(a?GM&p7rXAJg%kb3`nw!W6i3S><0q0aMVXkI4Fnd=8UHh+A|wG@xbz?VOppI* zcUnj3`hTr<_pprrd5E9f_@5uCcuxZM3%pk!`mub+PMezbwCZUouxFtzDUgWv1GNS?06*P4N3@^3!Hn zS0<%hj%{To+T|FxiPWjH0agjEw!)%P))p5t`SrvK-NjfwfAuJ?p>d2xPU(CPl4 z$KBRpH=F;xS^9rI#E)DOs@V6LfJmcw4<4yMxsJd}P$e!ExDQf4z#fOSvpV#Nct-uiLDl)^WRr4!bqf zY@8n3I!ry%?i|-pt9@EShfVp+$!X3r^7Zba`mfnIjm|V97?_$4@3OGKHcLh7(w?@7 z9zkO33^g&Vs$wKD1@qjW zC6eK39@tO3!H~3Gw=ussbW`GRGD8{;Gfj^O9gcu8t&?VOQ%#D=QT(SbNO*<4^i-$( zElKx1weV<`1_AYlAC+je@~$VRePZ+tU<5RX1Ve({hy;jJw`9|K(<8UB5c|U!jm7CI zzGdWOjBj3If!{Ie!*sP~N>#E5ZoFYGe9K|JPaHwqYT}uX^2;fX28@dVy-6-f`r;4F z7a!=K&}6syzpVQq$-V>E3SWWlzrn#2wnqSV;(1912}OAI?ZJ8K28&*p!aE+|@Y17x zO6~^K^Ipgm5QaytSDjW3wOS`N)H*qp3s&nSv8D|su*FErm2BB=M>W%yOXz7Q(SVj* zXdojSu%5kO-el~j6hw}FI>9)z?cZymI@-fm!#xTSj(0uSoAi2eW8yQH^qgQ4#T`Sy z*4Jz5yX>079f;_H%b%;YF>}fD&sFMrakS(9K6)l+UUCrP)eaiaTlK^vmONKm&4Qah zU`GHGTZ!8t82}ckrpBHN0--C-DI*^(d6Ik8qizjZSs*R9wRgXwUj_B(p8%M$x|vgd z=n**MiSB{a+VlOyp6_sX`g~`S{Oa@ksoucCD@;B4sb2q95FWt?#(Tl&mQx3N z=IKF5^i!6%?6kI_xJas9C);k+NOO#?7D^W^` zQkFw0dwY8lOaOj>ldW4W$k;+0cP_c24*Bc9KUYJ>#DIVO35Tvr`Q>l&bM^{8D`@QD=KKciOrVak^*buOXeARCM3YwAa@1&o5_%) z4siPCy>`HAf29s=1bU5$2&t2pHt2yM*-OeVOF*_XAj`Q`BC>l#WIprBeZa9ov~_=Y zEPVny>j3t28XU`ju{fh}iN-d8#*o88OGV^ivQCM~wjPs7ys@Ser@y9;oOaZ@Q*_A0 zZ3$TjJakL&P=bdNJggHu9Igf)+9h}}jg{b`1P?2Khs#Ny^Z+hMB{<99&M(Jd-8$e^KO~OVJOPD3p>z}JNq)ijDq1{6brF8<8u%UzvdM<7OY)Ct& zE(jYsC2TN_m9U|N4GY7D_4sruqr^k|wl3uwsz|+0U)dc2M9M+5G@_Jfs6<0(OVChz zMc>deo;Yf(L_;MS+7KGrOoYoW!Hy?ng~1-h{DnaYckyiP<1nLRsxIcEpNT&dqohq- zUMG2v&rYj>&~N;?wpm$tJTixPU^bE|DZgu z)4Ws_Rh!+IMwKWyNeR9h<}6@l24wGy@lD=U>gQZ7h#&@8$v&;}z7{Tqa!utqik@6k zSIw?z>OZ3p6Dq~TPvc=05NFC495-5xuB{oVFXX^axE;H!u4hLjB)&^R0;}`DW7fi; z9O-{a%sX3QIV4YzxWLY3<|}e#3yAx((s0Rpb1RQ~%w+YqUQf2!GYj{Z{UxFdM%2yp zPR)Fu3!FE5XRC^IE^S>*v09{0W`aGNnQP)Dn$1kqzeRhB4e4alD{^)r?%mxXLj?Nv zt$>N&Pt!{Y^=rw4B>e}44in&2UVl%9I^$~Ug%2Y-$iUrHY=~R};(JlD)jNAt!y$3A z>l$aFNG;p2r1cCMpuS75s5`-4rbtbM7$C=MOu!Akp()kTpE#FFeC=+?HCV4~kdxZF zuD2Vhr}Nk5XbsL$dAImx+7>C3i~@?)B&UbmhgLJe!yX%ck2%Q(8-uDz(jDoV%w?-X zGlepym5=18ZJ5f;tiR0;JND=>PshUbpUD;Rl7)f3mEdiY;4ST>x-58Wm*A}gZ%cu< zW)nR_yO&Swk)uF8D)%`k3FvnZLzMhL2`=XXmywTei7(fMFE@?aN^DtT%Z*^mjEm8- z;IcCvT*}cu=O5FRNzCng#GWA?j>LytJyGJ)-NvOlcifJoj>}EFw)?5$@-oNW8|GCb zzLo(`B2SHNS005_t2w+WgsdxL(z$uJJ_oN{Ca!P3Jv298Yze~G?q?;wUlQM2K3y4P zTVnsEsC+o=zl8XB96k=iPky6p)<-{6*yGOOd6;SGfSIU!7v~784g{Z^<k|8KIpLVc;ZuW zdoUv2xDGw!2O=coV2r7MpfBXMy>D|xJ}ofyRo^9LR`;W?G$fC|zeReo)!;$H*1%)8 z9Q_#L0roM%9`lEsx&&dwi9l=+-Et!VLCz$^BK~HY33Ky1_1-=TLiPu71Q7fK3(=T` z1W{jU7Rr3d{}H>+nMTN^g8>OS8iZ^k!%{Xjxs)d#*+(G>7?-Wy7V{5-6iOBfh8$RsCLzh;XSO?^ z%tJ17EP^x@K#{x?k#*H>+#C@9U>~Td=3uQc4liA+R{OW<7J@`x>|z8v9jIm(cW@q5 z3wuSwg-l8J_K~_QZau9K!b@l=sZsJuzNZVm+Y4l5E=sKZ#+xz6&WDig*7z=`Jh`nT z^j5>x-4P6Xi3LUR3Xo=L7uEY9={e)=MwBj`ekJ|On?bPbL3xSgZOIK0t>cZZnb0o(Hh6Ucdg%B z_8*<5eaG6xw5{J;uHwQM(=R$EJ$qDUeZ_A0$ZpvFO8e0M!*ZHGc5Qmov>17c8Jl3X zDfZqE*)`Xul~^u+VtU_YYP_43a>T~u|7Y(_yW6<6w9$O#ufU;l+j3rU)M&l!UPtke z_{M`;l9N6ipUZ$%Ac>=5p#q>J(~8b-zY9~%#hD^0fnLiZYrqWbXK(ENpw?2iZlVc! zXb*kg*-i?={37x8@TUvFQ(gpLqWJ{?YN#LFWAT_R;ZyJ^%e7K6ew(f6q(9 zZiqep&d&&kv~pFeMR2rMONtBR7X-h>A~16mJH$^%x-|DdyVcqQ?Zd-8(0X1Z63^o5 zfd2qz9-^ETODv6(%LGU>S+7NFOJ>AJQ&;j z9n85bBx{`ASnv`JfP zqA=i!!d?$M;GGY4lS|O^#rN7KHv$viHlzVQg?zde*c30WpDxZ{fpjJ8X3y$2dwvX9 z@fJ$%ow+tqvcZx|-|1OSF*j|mXA|Mn8nzq~FD4U_y~%}Emq{H&fTr3V$w=G1?A_Rm zQGHLK`}^VWpPA(`^~QZVfoecWfGEc(7xXlT4_KGbx8THr6LWw4 z#-5yC`aE*}z{+_m=ckkN+MYpg5bZDyDMzlhX+(Fhq%R$f{-wGnpAj>5$2txf4l$2_%<4A2&UKdHmC13%TYtH#!XWVQ16K4*-H38p4Eq!?Y<5d?9t?YA zF|5U~Gho;W1~_f*3VldNEA`^KIDZ8$AfIFBb6RXS!T6W1sWSFC#6^F5`K9{lmzCFD zfIby6HfqNs=8F`frT9a|V>Vbo+8 zHNcFMK83^1pCR-)_Rwy#{^G?RNL!R7@zE{xIz2Y>osJt)8SoTw^DE&r5n${AFUO5K z!)ET;dm!^)I>ef#6}r&5M!u`9q-$!5*FVnE5}N|FBpKADDE4dJY8G-W2)Wv2fMxw) zrDV=MB~WE~Bc3*I#Ab3t6tXX5Xcr8pO9KeW2X;Ra7qRDsL=175{$x6andARwiu_lf z3Gx4j`>n%F{D154@X*HpKg7qz|F1IsUq%DX8UJs?`ppd+)^FAJO;_7j8rHwGK>iJm z++SX-{)X$^HaOn~=VuG~_!aNHwPBr&in+J3?-nZQTTdD)O^khCFxuUQw?AIFZ^PT? zETZ!OR^PYy{P(9~+^~`2*0%A`+cYr1dS8R0KSRX$l*a+= z0f)HrQ^VffiP!J%x9U95v-5Z0QZ^R2P>p4GHdeW_u^}q5Rpl)WthsMbHMf!7(Pay5 zHWk|35_Js?Ei%P&4->QgG2CqfIR&px9VwJxkUN}T-*){)FB|A&CXY_SmtlAU>9(Eu@OYidwmGUK; zsn2J0kY23vI!15VHNRBJ&Qk5yYp6DK91<>nuIt*Lsnlm_gf|b_2*`Kkn!oJoztQQ= zQt`JBS#cMlA@Nr*;a|&yXPoapJ?OyakfUDct)TLkI_Vj!|MMEwdV#c~n=4uKMGhZS z4zo1IyA82-yowqTm|Fa|XmnK&#IRoT|9&+y-;=5P~_LpC(GJ{2^O7xM)R|H(@3^sfV4h_Oy=R_Lf30VR&U;Uh=x-Q7v+iZnAo7yJF3ZD9ec zf=qBFn+6Ni^A=*h)O!A=5?skH!6Nmf%dRJ#p`J5t2sj9PyV!Rqf(*@3YaQNf>JA+T z1ss)~o3Nw?In%kUWUpY!EuzD~SKA;zt4wC!Hdv>Q@J=-XRVH$N#jCkB+kEKkgsy zAKUn!hxpj|pH;^H$oAR5^B@4d2<|a^|Ln<=Z#c zZB))`kco}TvB+ebqjHuSi?fNrH_HpT+0Yo9Wq4ULG`= z1p_>6LE0_Mwgcc!-alV?_}l`BgF_guefN0NLOdB-a+w{TFih>2Z2hEDRON zS~oJjf_g%HfA{*bbAIvZ!<(z0-+aEjPdfUg-2BQyO3H#tW(+HtKCon5Xh}juRuVO+ zx`;Ux%N1O__aT~wfb;+#AfV@8O=-gVt%AT5X^vqI^!|Y$z2uPc55$-1w3ak+NkLm=d&s%Mjv4yX4gS2BYmv<$<>8tvqY+L&As$8@dzM%}%Wa>f*3Yp+#FVlS&CEK4%513RR@8#o5ohgVe(AtyVwNsh zg_hQA$)vTlmAO=8Zyi4sj&f6>)t+K1w93$HsrfaUDF;bxo43Il$bZJJ^dgo?rp!#6 zLg%?fU1>4-%TkNUU8cA{;Fn51=W8Z_{EQ+BK(vsq)ji##fI8KQ*F6Tc8n~{TVT`q=dG~Q zt-D7rj>`ua333Cg#Hv|lXDzW{j-Q$(1wAP*Xx=V6wx}gVDjc_Y(`weZTFEBx*kOD| zjeh5g2?>4I%uTs?$yu2)eOmOd{VLl_g#QN?{#*FJf$;x{gYE)=hkb>(9YWm)LEO1r z`FU`mEcH_a;sGVj*7cSu`#jzCyx@if;TD8H9uQt`_m+X=VyWG1gm-r*%9!Dxn+uY!0OP$< ziv&<+S;=!@v7NTFmy`6kH&hp?B{Mxp$I}Uk+Fj$| zGxaVbvG2_@Ks+RgSst$9*wH?LZUr_H0&=6-#@)%xGKwG_ip z;<*f{-Z1PbXwZk``*Z zT+`VcrL#%r+CE7COw#C8kKJ@#;u7W>+z=;)+^I&FbL>fxqqcb_!*F$tik{65l8~Q?Hy+KsW+nUP!i~q&HP=4P4 zQUrXV(s-a7a=->8^jtt3C!~P*-UtZP3j}5YoXEd>fbapLx0oT|43KjTdV~TW@moT# zP4x%`E@B+}a@hh{D9i*fkKD@ec#8fEF%{Xn>NpeV0Ur8d{Rt2SeSCv_QRf@%B3gf9 zw%)_s8geql5@LC?Y~cLkvh$Y9GRvEl`pju@F19DT#(7kqJtZSpI?~H=^J%d*BIJI$ zcJ;%d_RrlAa;pbsuyQSbmF2I>7j(mn>W@qfkuFS@plVu_eX>N9ZRx2j%HBkj{j{*H z@{mnHux;ZtDrTYDBHVjRF-H%fPb{UmOK7AXm@CS)Oi|hVt?P+J$tyz1R|Cw9A>1b% zPz0R;XeP|w9_cH~dDkP(HFA}aJhssTLIJu#K5&)Sn1$9kXx%zmBDQYNOgoiZZ{~*9 z`Bw@L>CJ%a{E}>4m7|`r;5xd#$>4g3{Su6)HX%Fk24YW4xQ?uQheD3YX5&4N+#<#S zbSPmAK+gjM!g#JY59fEBF_x+Z2C9KU)WASCFpv#wk>Om(@I@ov>Fr45xh*!y98}se3o!G@t#0>iY4XBJ_3&2U zv=R`b21P<RzZZ6u-e1i+QXv< z@YAk;+e38$L?J-|*~Np?*iI#bEUcUhR_?dQY~p7nHdrTo+;8WiC#(s0HCWIBvoa5W z1(4ayXA_hdb}cZoz{~=(xj6_4#=}TZ-opSCvEKqv$bbt9dmBVO-X2@`@fZ_CgJb_ip(H6P_#{gp z0Oj5UU_9~!<>(WXBTz6Ag@Q3FHJAiO!301G#)48%4opEoe-#5%Fb1T8v0Bt5unNin zD=^dXeU$jvqK`wZU@R;JlyvMQDZ<#G9A{?}9H&cI*qKOV{jjs(@)B`p z@-PNqai_(d`j*9=PYic10zQIOf(Ew3kifjRCqLdbp zT0m+6=_Uix5jhn6%7Dx*Sk?aAHBJ1*I00T2LxLDVC=^T1=XFZ>5;jw6}$%iBeif`ZOVF zr6A$$I;Ty|au;!ky>$5Bbf>j#rCV^jFe6#0!{T%Jtm?5Psz@UYdyxq1ch*d$X3i8-gkg3-@rF#U#-2+7g zXkZI8kbt=N_n?#<4Xj@oF{Ej#$z#H_JWU#~8j00NrmV%|(n$6!Ce5$W-4RC}hQH;u zN7oNaOPq3+g-iD>F15JS;?gIGOQSWRv5X%SW~40R$x|7guw{JnG>a00_lu<{7Kb(j)hZTNv3OL9#Snh&kQ+qh zfqc`GOH1Dyt5l@kTaHrk0e;29aA+v0wb>i{z=Mo~mRQ}aP60Mar?4h(Yw~`QV6)7O zy&SaJ&dPp^HuDcyv}w`i<3yW@Gv+MVd?2tXTiasI4aS<*er>Vly4@?1c7V+TrNtJ| zgXKgWrY+|dlUhuwZ&^&bwU~6aL-nj*Me^4}x%`%$j0`L$T|XwZ_D_pR*M>>e;=Fyn zzDok6hsEctT0ok7zyeYWNFN0t%`A2nj6NV3O*{EoK)S(z)EY^xlkfaazWGDoAF-3~ zVPQtf$u|LWpR$whSlbZ1f!Gs>x+7sqheD3Y<|!e?M)AWX#Ty=39mMJ&RtI?+I>@+% z&nhGjppaaMmKHil)CE9x0l>aiNUT1RDCGv|BYJV0LnD#mtlDZ8W?=OdtFIsmvZqo# zYJKIXTwgh~L5XAZ6^b|=bsQ4r)}hT+PnP3HzHg%zt$JeBlSikXq?p1>7rP`$_?D_9 zqTHLPh?t!b+1fz1w%rOzmU07B59^y_eRI}^InfYvb7aHjfzA@goMoX-OTuSSr$wDl z33VoyI}1Bk4m(A2gA(atVW)+iQ`Ex3PIdbc!p^O-bRI3o8GPR!NMbRk#hgzLb6P{E z`8)vPRCZ2zDxyW5iBekBX;J4>MxB&+UKcvon~FS3%$>`Eo|df5LQe}lpAz&;FL)My zu700gH@Mwu&zI7oPm4aEGWxuwn4?XFp2bGbjpQ6cu zY;uc1^QE*H)MC)5j6qeT=Pkq=h(1fqp2y1ppC=Z4TJUMX=Mw^-@mgnL=K}?`7YP=N zH!a?rq81i!s@snXZz|niuBcmP^IQ__v|O(ic3Rl^lwfCe$+O^dW#IElo-v_}odp2R zLeDir&!Ih4!m_l^$I>eF$@CidsCinC3o}wYt%+@O%hQ^bwthJ5AWBL8RH5~&7R3hb zlY`}G7WO1Nt65mh;wfns@mgnfij^C*{czYt6p$WJO>P+svZdSr1;g65t&TCjj*-KS zk6Ohz8KYtxS{0)}ylt!s@nSg++3gorg|I4wRUsae3NiK&Iv?F4#^v6`H5!2$82dEw z+{DvcsA8nfrrSha<4Sjgi+eq*S|mzoRg0&oTI@YJ$T`D$y?N-g#CK*{@YKc&T6k*V z>7#pY4pYBc>7V<}^xfpzTWzO)wOrFK|BP!Wq;rhwexEYebT&unY|^>557Ix~9q=FE%tMqHN?#YUL6<=4W@K)fEAUuOf^tb;+b5C8LjLJr5({k}{F5Xi6dRm+JlR-*-A9^$4Gu=oztoq-{9m zR+j=(ie7$FPZg;|_j&ADY$%av@2x`QW*r6NOegZj8kOq$s#GsajF&z@ofNr7pk+6|p+HDZ#LDYupw*_L^_(0Wo}H;Idwx0R#@ zs*4;vRC-O?0YA1Q=!qTr84j$f2fN72m9gBwpqII7+HO&7HYqeiB*e&Faar>=k9kX( zdt4PWce${fxoMF%j-<^ZdztI+(M?q76jsqqP=+BM89Ye5n6dc`As6~dk-;m`s?1)&XNqmH=b+EsU`+W-^Rn;c!=C7LP#p z@Ge3l|4zdZ@aY1;n|?>kVF_S%4WeNq#?i17P~srQ2<`kCq7ZdFS&h>4#N>QFH;jp< z_hcATB3BBl-+H*}w-&B`tF-z*&RhKl?Eco`{og7d0IxO#$BJ@-vl~Rknqxui93Xb% z@Snx0PYb861^=~5e#JV_|^- z0)%yf!H8i8dM2eFMUzr^4jQ@KB}d{6IXn5BCuBM=?vN$haNF4j*#g#l1{Opf)_nG; zU`d?J-JOeWuwysj;tQ+Yggc%Cyc}BaV!_Ll;3YC*NN*Qqk4QidHQWF8?=MwK2=B4K zeumU_F}wbic;OJ$|4(WcZVb-wNMB+M@oqAM8<~#CU^koeU%#K#{`a4!_5bsHTqbH` zPnA;uqM`~h{B5-rN@TFy3N5f%87jy>_z*zvN?yn!f<@WpL zL2$;aEeGKn)IpRQoDiFtJo48K;)D7Rf4RiDMzsFJUp)`{KJw~6 z{N+Foi$7gj|6!a>XcD{-FeWOSfigbgod8NfXNu_R10{4ReeC@0-_PnlUOcHd z|KZp3kMi$1L!nFLS00MeV!vPi;o1B}oF7KYl7lzuScX*s%4~m{#@p~NmBrh_9#zNr z&4_L1>QGLh&xV+9@(d**?~>5p!Wnv{TX%*j`f6EI)TJcY&<>5}r;C{bscUBHR`jWv zqpv9uQgfOvvg}okjtc7S*8-3P7WGzJ2LZp5d`Bb$I=CBZ*SPPN05Y1~Lil9((FKP8aD7U-y zq;SvD>_8jTX;0JWvk9ox?rGXR&Bu;dD>~U6!DjnNUFIWCGltP?*%A@srh2b{NplQ; zS`3d;EzZ$b+1}?R44HB)W*#U+o|;N6Iq^(~RbN96!DSlzCaBX2ka3D6g!z^-}b#DRS0`cQj4S8o!pIWgSb(nxte+ z5wfP}Sd(O|NmA4lC2ER>HOY>eq+pE*SkpB%#HM=f$a9*0BrXEGuO+dj>f0=@X@YsP zraX`noTmGg`^tR$k8|*UPEY))Ka==BTg}4^|7W{()U^Dc5Am`5pPS16X)R#Ow}4q{ zPip{MVN~I2={@&=&}d(NsbbeFGg&-lk^)QcnJTain0peoiM^);5)v<3f=^5EX~B#o z__PF{mf+J8d|D8@(fH5e)Tf10HzVNB5`0=IKG%;wtP#zE z6AMnF-kOj~Xsniq3R_Em(SrV98=4OR;G&(S~E9?V;G5KadK# z%MN(kbSBMt+e)%$Zg_l|CCs$JWfo4E?zC{q5?)$7wK;fdccU0DKV1Og?>+}N$l-+U zHlIHS{|D@DBJpLuq>Bd5D{{TXk>7+KeR!^}MLP9CSrC@rr{j!&Gx({Mbm_WSFX!mc zHIZK~uSYW+m{WM(EjCRbm?CkJIr`xAm&>20nl-VcS7#u;oHSE6zgyQqfVqNcI=)9| zpuU{cqgx?QACd}c1)Vugj9T)`r)BN2?$B&kZ(o#`cDoRU5soq!@6{Wga#6Y=3>Ph?AfX_t!&j3*=e|-lf zzt>#U4f}loStrtgj^+PswOR-J%?$tNq4;C@KOf+;BL2@bn#c>%6q?ZMOkeSZ76XY* zpbeGggly$JJj7gDDw-LX$(_NOKgvt&EMzNarwlJ?x+xZ-FYRIGK<^(2(n}60|3G|c zXHlf3g-oUy_EHbuppR+W)&?+|CPt?oP2Z1!ZN`4tTEKpJlCfW=ywjG(J-R;fQHuKm zepxuXiuKdAn>J(BAO*`2oQmO8NA>sq#?g59m5foq?-JlK(aEGEl zL&Ug70O9b?y!f}5U#cwZih19SNlH^7W3D_Tx|op50VE58A3bfRt-iuvP{GATbS+2qTPrd0uu;};fxY@Okv zbu^iF>OuFx(P)t&X*DQNRZaNaS_w47`Vo5%4P70|CDgdJX`J?Rhf^N-=MizpyHU~KnVF}MB!QwZetZ0-Rof(326+rNHP@M2a_aBw&WD6l%W72i)? ze2)>_QiT5P<(F!^+5A%7i+{BiI>TVYCHGs1J>-Jq=DQW*(~Hl8cBwV}0fqQM+gi@o zF2rZHoFC3(IiEoFzg*4U%6Kc|H(SP!cdENSVV!`IucHp*greO3`^;W?V-s9XB)}%Y zgkMe=I2Y9gm!i6JxaC>s0Lp~+(NTLtH{y?uR<>bb;l#?|#L+w(1=i7j>+ymO9b7(K za6B9C@8LZuDzh(CkwhuEL&%-y+vR?aJ4;rkIVG7ic zqX1AF1`c!v$nB_=Zvnx5jeRAlHBXlO-lrrCMD>B+M2_+Vv@b~8xi}xb5?p*Stqb&^PpB34hXcdKkUHoNA&8@3eOH zt&%VZPe0-TKDN+5&(y05bT_DaTCZl946cbvIn!x&!=t7N6lsk8dd zZOUd`^b+rSai-yQ>))iZc#h2w-z%yjcxR}B3OfZ?#SWxszdiapNZDT#Qv6pj z5YP{Q|3-nK23uNgrz1z-;6D1jx3LEQqf<65^H%+um;yVj;~D!p+x0Eca5s_g4=;k9 zN8}%_^)tJ$0BE=RH5{>>A+pP-D?Y*--%-hXpuGAzmQ(;~cE4QYEBshu>XZ7Gj@Pej z7sA_5!#Z%Inx~xqUZtPhD(J1`x7T4lxJ11!-B3lms@s@}-NQ${M)|i)9$`i{TVhLjZ|PRKG1Dpm;`e_(`EoDFVb$hEb~e06 zSqOqG3IW6<4gq&L1)b>3$rC+RZIxnCC!~&vC#71uMZ5j$GSaugx^FGvSrpBrcl+1~ zxgo4)7A6A!qXKfMmS^Z6%3DO^YcI}C`Pl#YyCEI?=kNC7Gh@w_{r|uWk*390BzEG0 z&+X6{z^k6iZdYUKdQyBwf!gs~$RIl_Q4^8};QYqJ-<(HEWWc%>%D<={+7HIM-4RZteSTs#-^`xha4A+f}3o!cCbv|bzR_k%*<+<;ol7{u>jSh7?$ zKF+4Ni2Wd@s~3B5ZB@wM>73}uX?c$05@STca$?NI0JS&#?IfwA0vZtCR0HB6I#TM} z*2#p?O;LK4QhE9ZQuSd!rpljHL?wST+28M-8SnKlMdEo^vxetnE4#SA zuN)pF_40qAj9gb*0Mr`|-2q3D;BCn5TOVa^s|}MGxx#9>5aUMq$~)Ik@QFcA&A4>b`pj+VFhs+q<&%9QjPGhe@5nxC4JM@z+~ zmi}al(bLZqC7-m=tys&hC3na)D%#GVaK;vSQJQlhFquTIHj76dJ;>JHngijl)+`PkX#XP*3I}jG}20$N}{|yy$s;9uxcyd_bMl(&-~_f|QKD zcumc~#5Bpi0I|8uR7*Fp;|!Do$@{OAKh{M(exLm*56}DKIL05lePIfmcX>3@QsiG+ z&@@8xNyGGyVKCrS5GB!^#uJy)1axHVaRU-?X7d@VUD#23f2+)}W;sJE&WD^LhBuqJIF{bYjRNl;q1y?$U z)RFhc_3b#&g&vcIvH zEu@8*DX%{3MG91N+a;BUQz|q>B~vPB!Ff&41(qC2dNpefJC|ry7#Bu;+F2UhQ0`d0MaVCW+BW7em(`XsYVm1|Cyhz{j)^hlIVWEq;cTSvPP`~Z*AfaC zYlHzXh(s)g{`ek3YN>2_L;}@9xMWcE@%9I^B!z!V!TKNn*7qJkE(37@BWgA(<*ZpS z7I7Oi7Li#rYS{U0(hpsJsu_`O!q?7?MY_G>XlOdl$w23gRxy@jr`d=o^V{fqy&5*L zCzWWIa%;mNUB#04wVO6$6W<2`8`4g8YcW3~67w&Bo12^8+WeUqDQ-PH?Q56xTe@Pj z{z80t3AU%bwRZfoG)Pj?)dpdm$YkY`fFI>hyoe?i7NHi5Nf25HWN-3U0fMFZJTww8 zfxLB|(5(ouS+(J^Rf4-r>`A zzZ7pTeF~*9%c?77s~{k@(~_l0EN5S1w`TSqs5RR%I3K;bs@4#OR&c!9RsFj_ zz0N8A)?x?j`bAr?;})fLPV03reYZg{eHkkg&?^(ru4tzKS^o+vybjWlfDHGZxb!hR)y!#@e$oR40H=UHkK$41 zTYBo2Jn--k!R_v|vHaXPl%R(9+OKPkEdrGFwxfjs&7tW&nfZ#Z1d8_6kB_f6 z<&S^Je|uH&v->rhBEcnGcQ;Iq4k2vsxJOip>Q@svhR0aP+{}t?5S;y8!4;~(@wu)K z3}gn>LQXhhtipIhKQcv39<)!d_+kscr%;T9H}b4c3hL%wBrWr-NSY{`b+|PR06O>Ktg7)bC>t_lj}0ytfXskLShN^NU*kn!_WUF6`Iv zVC2Zd0oqB+ceK>z-i_;3vW{q(tXUqy?LRr%aCyg@^hgj1}ZT=L{ zU_Q2sHMLV7Yx^vh6$_?YO15t3;xEnPI@eL+Z4eXxSRRP287nGd{w8lKeMMt)3J>V4 z4sj#e%&{fYV$S;ceBL|Ec-r#Pj7FRtwVGl$CP60q&1zZkty3HLb>nBrOlaFUC-9nJ z6_=A(HI<>@T^14HU4Go9g|^4T9tjdX*0w7}mTYV5HBUXFIs5?GHp(P*QJKj-TyLDY zloIBfY&CoN&9I&Jz4q0~7A*VUsrFM}ldsPDRBI^_4f)y{TwJZ(XUc6rs7SsjHb(}) zsZVPVNVG>|?WNv1?5jwgr;}_$P~9Y=MYR9Yn^n8C>mT-Y_U-@M_-=rFv-+Y_9Q`5w zO*)khpg?n!eEQa^x(6!gssaqaMR|%aNo@!&K{+r2;b#+`Z;g=qNWT9+q?Uzf9}F)T z)_Yc=72(>jH^IV+BXs?<#2F6*X^zH5aI|p|Y$zb5aFB#J;J`&)5$av))Bm)hExT?1sr@HtDu~fr%91(I->Q&pIJxp( zG`$nbajYsH zY}f)HB=|J=cb?IGgYPEkpk)DK0#{9Cf4cOkOCLAT5oVh9au7R!~I^%B2l9T=pR-ccUopJ4NF3Yla??I9&kM(0e#GP@7+xP zlSN`}p#oG;OILCl2V!8qMqx>;5J%B$sgh78R)i^g{i9qVY2z=qr2 zmTkeczqdkMI?#6XU&MR!PmdTz*fBq?GRQ%tDyOKQCaD z&>n$*m5w9+c!Fbj8Jk9GlkJ9X9(cyEVeJdpB)cE>coL`<`$3c!B@ujt zPG;GV6KMbyMgX~B_!f`TUs3rCZ~-yU6F56bEN(^yl(EW2@+OqOM!ihDm}bGPxL}1>H7?h4}c0|sj zc{?*^6{|2dXD-O~EkK<=6GE}!TRc;X^lP9?>=YQCG=l-|fAHAvZD4g*97{TFC6#_S zcE(BzZ!9aR3r*CRt`2c4&n&a@ipQKzA<|>G^;xmPfjf0mM0W{ zTK?AN8S+r<)7mRQ41dl+`wK|n*ptQMqY9Sxy-_wm^Q&RLgS-(d4?xl#JM6=*!Z$w8 zNvhM;7Sr+<)rEZpx$UR5>;vI~g7Nk(s6(4j==X10tx@Io4aV{$201s0D}2K9a&)c4 zaaZV1Y$HzrqTL?v^|Y-7ufWrGu-dzn7c8du6_{l#X!^1+Wv~g`ACq=KD|Q<|tBt^R z*DaM3%ghiO6GtCzMidS?Ciyf6g$9~9 z5DHKhh!S|0pFhPX1CADeAbBVpR(qIW6(|T40Ejj{O-ji)ho_T z%H9ZZgezf;hE%sZQXtCLlDr;2yYaw8&3Ma8!egsSH7;hbs|%Of#Dr*YYL4QYYY-Dy zI*vba2MUVHmg(>Kz>s%h`|56j5pXET%!{O`Nf@ioK@QJBNaW-&LjC2y&|E0{f1RR3 zD@|KFd8x*a$4sYM3U5Tp!p3+Ze!#$Yo`TvTVk#yq5>E6Ztx$R+de;%Fp9G*h&`&E$ zwkz~-u|8;o2CyMKA9IvZnjzD)_Zv41`S%{7TP2s;iee@)aqOjwCFes1aVbj&j~o`w zwm?~vbas6RieHFo=CXX>VFGt@jLD$$pmEX9;v|J#(fS1XP|~xDG9~2cG1E6*!Fakj z|KT=xs7Mi0m*nZTWV~COj!1m~yE#h7#w===M)12CNWPAKPy$+#P8ZPVl6=b?jdJ*R zqsnxh=Z5F&Joi#%1+>h0$i@1L7ZJN&o^Af^uZDBoC|g4vGFiF?dC$6;FvpHmsD|uy zE?c@*VYN?l;EjVi%IhX@H%q&+IiX`0CEiXPWP0(_$vje4vd#)RZFD@NE8a4Z>a31= zSetV|fjt4ByJIkrm54aR)s&LL1uPFt$wNQO>0p_q(83P^_qhU6G&2>iOiU-_>Yn*6@tio|-$pOJ;M@YOG{U3v%W zE(nTh_Dy>`z=Nn4fe}@q($jcn>;kcLMu{KT)uB|IMMghF2OmQ@5lX;ZD&c`@XhhO2zEKeb>xPPHER%`; z0uqb%S4Iz|y=cN|Sp>>H@gI;X-s2w^G+8%P;oh5)03%79EQh}FK^NaLrM!NMjeWmc z%cR#g=$(=_7bu^Vxa6Cs8Yu>#U?Ogo*WQtiyH<>`w~`7K74};MJ524T!e#qQaVPJd zX2u{c!fxWWOzTr~Rwei|NRySvDFWKmw#xEwNUS2b8r0mnHf=y2Dt&U!WP}BkL(zfl zfK&pb80j0bgH&T#t1=c>Z%2=Klr=3qpW=;1wl)x5@85b*?L!VMgT42=J{7GH6<>gKqCk3={9-*4ba&8we#3H+*V}-30?9+YBct;m zb(@QncT=pU$jNz1@-6tLJYAy-DH>Dy8NE%^VmO#$Co%y>fpsZ)c3Xv#RFNM&xV2Sq z*zIT5FGD!Vp-Bx$vA&bOiHQAPvxwsZofK1c0{>7WNT=;*2#hW(!0HETDuENz$%KPA|Y+eH(%k>7)=oOJkobIkK5PO*Md}L>M zCyNCR`l+E~Y3CTve&+$HPtCt~Ox3dohQ;~wA9#{!mkC?avP|!go#Ph^$S>$eq{zEn zD_6@oONQd-J`3sv@WWXSrWTvF z{4j^#U&bvoU?#W~KG)`;^G0(C_V_Tziz7-`Hnmsw6qe&lG|bwGwD}@@1d)2}0_e2V zUPfS01i`vgEoO9b@0|E-NAwq?r3kBI#a0FVSzpIme*QyqU*A+K!rN!G0vf2NFI7zq z+bJ^=sB9=GrKL+H#2C0Iu}4-gKpsJ@T)oEV&PLUVjrOWFy z79caWxW?#X?Fqy>XmzZRQgCM+syy#wmSY9B_%XEFnQ@A`e zNKFAApH-mQP3Qj<0&t^U%!Us#-spE(;`pj0$LB@;Hr4F`wK7n6#jB8_PmDg631$9c zHaa-=kK?>&N>}Mj!&R+YjlB8mR!B(io#F!N%gkO{qc%$>4CRd}AOel-0I%3qw8+C| z#w~QruX1tLTae*O8pG~jZ z3g>w!!f)ENid5v<&KQ;z6M4Sp*S|AD3)K>X4);bA^sq68QFcyA91%^H;TlvxzX}w5 zQPwkk#z!N&q;5G(7!4FS2OtN`uY-|Ax|aULmzct?3rV9(O4Gb*+CnsHEmH(9bmo#a zJh?LOm17{OK-BLt? zmd=1$wO^tf?bn^l)Jes~YdWC;4iR=uPZS0qQ9uJ9T9$-DNQ@pNWlnQxJ}AVg5eCoj6KoCTAr~>CE(3!#;Sv79OTPE()wvLW@L+` zggVq0N&x0epJfynyf7e@Sb=&F=p^Zdqv~2t1*KABxV(Ur$JshJ!?gw%*c{Ch>PnGz z-lZglsZ5niXQjLZpNWUa-vnSWiXRx%OSG!RZ&XUBQB%59R@Hgpr?~#OYGL<{W3?ZQR?;^HwHxV~^mv zIZuPs0j2-(e$pE6TrJC|FA0wCNaGBd7fhs*)riwBEl-7QNg3DA2r$OH8N0ZPa@|oh zOEaM7JH6sDJx^17!eEw<+(y24Nc333R;IRj?>TsLj#*Z_Ew@T6oXX!zHMTOEle6=R zJn$DG*PF=76Dd=TfME;?g5O5csLh^n2`R3Zw2qFiv7MgP&{) zs@4JN8XwH~Bd^8+pL&sf;I|IUl+yd%K^W1c?2*2ml1Qk+?_e{XP2^}vk2IAwV&uX< zR;-I2RKaSSNO(ET~mysFC^Iqk* zR7mfn30=s9#Z#i#n=xWMoJ56UBCaGl`rIi=O1mX6(UV;-NS!2!V7On1WSG7&=8e4D z0+@0fx$_@zHexv;`f6adL>5Y%@cDjUHX|Pw`vR&x=x~ZA)H)jp=Q)@jL*YuWD!+JLlMwfi3T5PCWEV=SWrZx4;lWPjwpA==>y z{Xao5tJ}H2B6ygE84bUWtyf8y8nT4u4K^g2Bx6SkwY3ch91;gj3viBm=z}zLLa3Ep zZbQQ3PD1$xmp^!X%I6cC03y#8{bTx0#9N1jKrTCLgn#DcI#3o?7420lH{uaY2QGMC z5g8N(cp1x<&D5bG$2dtIZast~xS?ssi2HN4AT>%6f7Z$5RwE(LWUVu-l1**xmmYz! za3QC3t@H8^25x^$lOy1zI4S|Y8!C)>ISX zyY05tl4c9h{@aNLGuj+;e{~!Z3v$#Tt$QFp@2XS^_ zQz&*D+kNgx6peVh zY!DWmb1!dpE{2B2ctb)uu=P-yMjc8KP8lg2Gn*VUeIF>%wmu(7@fQ0|t0OUJ>=~oJ z$I`9UVulVEbf{SWb91PpT{~V`n*OWkkV(ES*RC@^YAudZ$8=&u@~R;7kEXBbAxycIcN{TO*rXZS{gyeU9x4788%2eS=MVcvJ0t*## zBZFj;PJfTd<5U^TvE=EL11PI8KMh2sx)5nqgu-B{s}V|&nkP&ayvF=6G!ATQQ)pnT za2vE=228xs1q$BY`^}TZN#vY8Pa{2t{}zyHr%AH_vK2amP{Ba@_IUPE!|kH>{)+XL zM(hY8a_{FN0HN)9QmrbQO+w1{9~k&?LHE{wF(ELPk8)rw!n`sJ}96KRM-+Cg@;avYRH9t0BZrXnc% zrgUEY7%x4{aZ!R^Md6QE-Zn%jg;-2Tlf<$#(Mj=A;y*SukpaH#X<(@ly73TkcGjHe zIxFw1MxWxPpV^y?Bme?3b-xsn*IwB;6gKIqWn9n6d4bXi{7Z)If+guV4?kGVk`zXX1>L9hfVX z4~WMi!SkNoQL2{rzOS(H8?H}uT%~IMnvNxom>p{Bb*jcAWW2#D7$62 z5s&w*)4U8XmPb|@;X4)Gi(Fa!tQbdIHfxwKXeoOQ;662MouBAa7`MwAlkCd(5)0I* zDO#UqZPjgB&j$l?kRa!e3+3fG<^wD97PbEzHU0btp?Q02RSq{4brp?fIVwt+t#l2O z@H1<`$cl`r@`L4Rk^TH`PGTx-q)nDRH{&WzkR0~{>z9;LKm{_;{YQh}H-{Z&h0=2%o->OPS=J=NWv{Z%aGI!PukkFBYwH2(xO zOOJY}JVKlLm6`vuDE^D6^`6MEIqne<9m(~!&ZHTM;K(aVza8bm*(rxjXJIgd` zNy~6+Suas}9X@Tt(uZWM#`D$GAdCLDKII}E&fdH3!W{7a7Cqbm#LE*)bp=^}@;Wbq`Dz zdXzrc=nt;&c6wnyUn&hYID9d@lp>B$>ImgT^6G}%KBE^?Xx%|-T6=Tx%N`jF(6Tpp z6UGe;*qPQ=dHW91fGjxcFCZRD$b}4_9Q_=-ch`9zK)y6~Rz7M{>MtFJ^_(JM*ld0=26BRa)QMW3#Xv0#|S) z2AN*~EB4oHq@QgQ^e@`x1LJBcRE>a78WF{slcayM$%W1AIc#1a0gA$-(~tz3n-j{q ztW4HeEDi6-Q`v1NrRrsY@4M^65r!`9CWpN2=}eS(#6{wUpXT2~{O# zJ`#32LO>IrfF+NCRx!+i#~nWTmm4WS0=ns+@);3f%wO2R%h{lWQ)Ow&8nsojdzL3d zk2tal?DMZu`P;%L6~SEp=O1z&zOy+%i>14Cu-bc4J9$v?2AV*$fsE|OxnMPw z_C*|=^Z-brh!*VJw!noLNJj|rsTCI5iF=*S1-rIui4i~xLk-2;8)-6weo=q+G(p=v zsk2sl0SC60ki$QUUlKeD(pP$Pdqk#gYkx!#XMp5((=j3-xFU{fKoIKmdMeUcLx*>M zDA*~b*bC!ru8x5`i~}>pGPz4L@ri%V;)v*k+VTuEf}x_F>mgT~LcQggcu<`Jyt$BT zZ)LXB8EUNk@*d_QBf zYTFuZ1suaVc&9PNvKocgW=P1H=Fbf`+}1aOjqUAC%x&-h+EP)0pZsBcA0JH?d!1K5 z-`=W@HOUMA&h-bc?R>XV67cLJY6*l$9<(yNwvrX6?7c=p?nrlGc@x15)P)ny1jl&_ zmE@(|i5^91iH-l5XVpYi%aF9h^dT{#tUmrDDC#M}-teqPa~p}y=2mSv1cmS-tfIs2 zt~x9$M*3({B{E*hTT#td&O2xek?_;Gh?W=O=_78&JgGGU0LEf38g^XC@Ja-n5uj{# z&_{YHP;plemUvxpK+!V3AufE*meQ7PI@wCWT8-P<hphIvJxaqj<(|CbF#FN>p zYWqEQfwC4Z_=#n8bz2s-j1eXNDvAMY1K;Wux+`1(2p$-QKfr-cR1eSsV9o-;2|}=7p;$P0OSio!Vo_CTnFi zz};^|5gEUxW)a{8KY$cX-3dw;ovx}G*QA^kO>~SFZn`07b>o7@|A{r@0v--rLPO zZWEFQXt?(g)3peZ)hUJ2fSy!BnEw4re2h!!EKh8TyV;y|YBi?;@?_)(EZ!@2iZsjM zoO+gQV`KW@j;4cx_rzd|x;nj|PS5=|hd!msjc*F3CdvK0QobtNDPkvT z_z(C`f>h5Vi6Lz!QiPq~M>dMdN|(i~e_=qTj74xCN2=q2e&Dy0+~Z?D=ox#?hUw4ql0d71G58H(lr0LNb7*R+ zMCIP<&oYFj=1h0Vb`^{px`uf^yC}F!Q?Shg!+L(;)TJbidhUYctO$zcSm*Mf#bqCvK2#buh)5m&5> z5OYev)bIMo2nM}KTEfJJ3C^K#4x!{Qnx#-Y-CeaR9nBxu9Ncj%bt_p20-!pXsqfnL zgn8Y9;m$J(T%4C@NE(5M7f)s$FZWz99ee3D#NquGJ@}C-v~V8o+CV(UXDR*Zx3{_0 zm5(F(Sv3!we;)c}MZJ?e$%2V0Rk=PMt}lhj>@8_bF?dEk+St*A9YmsJg9}#9!xzzP zHj->1CdwSh1tVOL`}FP1eNIkWdeBpHd^cN+@8m|HM#pnuN_cZmVQMg0S%ms zRj57nj)|ntXWFuvo|zX1aAtwLC#6fiu8x1y-*aZ%a9USjOl5g2nGM17B#rQBg@B^T zB=KJ<;##yMml8OyalqY|BA~)rg`qu&8c)Cn`H|*JMzaFU4<#CoZ<3V@li0~~K-63U zOii;ImZ%2ahlN=Q3q4a!NO5ZM6UH4cDKOK$C3JaJl3Ha~%2cF%q-9oLQ&lo&zgP8*feX1?c@{*t5l z;=QT})_WGbNE}Y&n9*4sf>cCmt-hR8YNxZ~6{_8t$|&i07xZajW#Uh44V7nuKhuB3 zKykqf%Ob4PDjLs~T49xBjGNgxDpk&osAIsaq!d69qxkitsUIsT-$8a3GAUfW0^rM9 zS%0v+ea`60OS9u5|AZoj8uzyCE)l!*Z0ZWl>pZg0^jxOAR8oq{yhL{#OWxG$}wrPCIDl1?F;u)e# zZ%1eDHp7RY9hn58gJ>k+{YUXa1qXp*HS1(3fWlJH6f9AT&ZO>@e64=Tow=c`cbLdH zoz<45B0S##6CHDt8%g;va5#Y>sZBaI#o)sES(D6r>F$x!UPry zJsX<^JHuaAu;e|aX@w+BTuPBRRqSOAvozC@h`sW&oR{LcSN1l4ySr|*K=8m6rqrdL z3k($oyDB>{-l|MGxhtwU!X*JiJ2Om1E57us$cy)+v%+1CfHA%aTv4NjO6%gmD!WjM z?m^$VJE5~Xi9^fVJ;DjQ*3Jrq@^3$C>HE8UFYWeQj1Yc$LXqm9qp&itq;d~RU*h6L zT}q}_B?^FA)gL$r@ye`pNguB@E5X?l_60G1e9fOxc$5}us?|mIJMKX1eJ4CO_9$gY zw$tJh-`|E`WQ>iQL!M%A>5%JKwoDz7i3o+z(IAIL3iNXp+4Kt-4t-*i zwFfgSLR0ffDQ^;PWw0lpd+j@-cFmQ_6M$oqM<7_(&Yr~-cOki6i28g&`$RDgg;Z7s z3Sb}+?vUk;m|cl|5K=EuLH8IMqAxPP{eZ0UlLn5;3q9O?V%ZuJHu#I#wHw$F?~R^V zS!;mfJV0=4)N24&8@AFb*-cS7v?mjOo5bt*x%l!=MNz<79ew$}AsmYHI9JU&ZZ*ER z!oAVWyL^2bLgS>8pK57hCM&D=1Y-1Xz{nuIJpk_UtX937*lug@-lz8NLz6lY%aHj?I76W?{agwuaMgm zozJKsQhSt)7u>T>Y4Bvt(Veb$2ED6a>mKcwzOgiv;=O8}&)!Q!oOdwya((iUt+M0! zEvm~ZChzae7xeG&oJXZ>KbQ3N$mGqtF8r-bY)tMJWw&rZWo-@LVhY~k`R3a3Xa2!9 zR~=5=y(oGveSWXIzJn{!#-edM#hk)?bQ?Vvr#PPK7@|v7nYjngRl=( zTCjym7d?1LCwXPi(CEaA{WtMDSH4r?H770EdUp?m1;_Xu6odoJ;T{4Pi0L|>O5bZ- zn1>!dl{Al>WZBou6Rbc}vEYOkM%?mU4E#;kTt(*rMhTAQb%tB+@*?b8gRA?e{Wsi> zh-0QVSXYPV4=il}4N12?x~(B9ZjDrBAdeVBZU_fl+|WEzstD2_L@u!$%HYWF6i4lnc+7A4q!73%f(IO)!ZTftup z5Cl>nOPO2yH(Dq=QKyiF`>?d9sBahkb1QTq%+({l3$PBsj(6S^`7fP-V9XUpMB>!^ zwPem$N!fzMte|6@fBKx&Z%cNldn-cT4Dw$!(qLzmlcU4@Ou&Y)80n>OBH~?u%ud(s)Db(5Rv>0kKP^ zOR*BuTiO0g$meQnQM`^aD!=!GmL2m5;?+1?BZ+AGPiA3|>m6SRvv9|iR%Q=Lm8p(g zyJ|T__DtW$d2gvks%okCY8b_4T}wxB_5I3P((lKuHDp$!CU=ht?ZCF^ zw6I)Jmu$cp9o}OMcL*(J^0>S+1+&gfk~++^ZtR;b<@9prQ9FRuu`m@vvV0iZqd14O z00Wua&6Hy&I<{mD6;_c;Ku0VqK5k06c$BWxE^{F#L~>2w@0LfF@FY^~KqlK#sM3s? z6}6TB)p@E|*SVDgb06wIU{pb%8m55W?O$zs&1`tFRbMVDk@WrYFEBa~h!tkzw(4!m zz6S^Ja5@1_1fZpG*pL>wtY}Kjq^%Uu_CA+B05x318!#W(LUTgr^ds7NjZi@}Y#+sK zKCf}=gsJfov$$@0v8J9H5+qZ3wT;(M3=i4`PR~?1bYY_U{0uMbuzt3{z1Vt1n-no~ zHf17jzPVC#gj7Ps*KCoq&>EZOCK`LEH^e1gu4JCuH_9FkE<$Q39z6SJ*+MC`3p*`Y z%AAsHKS;PJr^dwbdS2d&@pCnrCB&Im1-Kef!ZGFictP`ohCVSZG;M-D-skQ*3o;m@{24P#7O0u7norIgJyVtevYNG+uLuug_FLV|LaH@(pUg;`w9-B1 zSfjBOz*dzVLuid8qWyR97^&p3nMprzk>8d?IAvE-9_`BU$1v+OVYr#3`XTGe!u!lQ zf~r%tzNB3pLmf1c%|jvu-BHYK8d%fRg%Y}Zy#Kx) zZckAE3})2g z%w3kEvmLB)F*$LOKUPuS#(gk{o*1_jlDN{#hht*$Iz1ZpiHB4)e2lV5S(#l=l@YYj zjriNjwpPf*PPtyOQ&QNv&K+|=Wk1Yt^>@1bMb&&t$AFwtO~sp{$g$<*2pW$dwMS+- zA0yS{pFV{NNm_IBwiG)w+8V8Hojb|)m&>CmAZdf;t`tmFpWz21OU`vZHmgo*!$;~u zOTZn@WyYmb`OQZ2z`{6Hs~Qqubxpwt8AbhX0TlXiNnkb7mwsjcP&5^I2U28!i=(rD z*pM1Fwa?=UlKk}WZi%2ZWL-u1!w}P4gvQUEYP)La^v(jN#}y)L`OM0;_i zl(BScHtCL%V&e(x%D7RK<-=NeZ4UaW>M9?9K^47Hg4bZ22{O8l>Y=5SUV~pvi(59`i zoOMC#ZBDlCk30i*g{zY9W(#+VOhdSmDc1Si&fp_Owx``|#5 z37*1xOPaD1wN*q_Fa!Uj&w`E(E}sQ>FVRFvC$@}`e-UOU=#Os4!8?pUP0oW^Vy*+D zLkyz|iMnKAUqmQ`xY#~3wMh8)tpP+Fy3!f;r_8xr+;Hwe<^QAd zvH-%%ekvgou&B_w`<2TPd*@;kG6JGWvb7Fe#cRU2pQNa&RLtaA(n!q^i}9whO0cbu zT@c6{pm_fjYpM)UHT1A@lmI`DLhXD02be%-zneBM;NMe%E1sh8Zk?k|B&x(z7(+<+ zRXfi<)0{|~gff`?Wn=EX@`L+Ep3NTk5iw~&_0kBiFUZ+PiW2bn_)-MoO=Iu! z<*n=wcRT#_5z7a^+Xxy{Oyayg1A3g8@WR4cl?FPsv0Rn#m$RCkT zr?b@)Du!I1WEJjRN>?Zl2c_V_3^4Qsn|S~DsMYdWM0n1`3SGXGWicD!NL#uIE?Oer zgi0BvE{xDRGE+lYl7>fd%cM;Du5lEYa>4f->mUvhq)5geKt>)8 zBeRH*f)k_yQVT}x>guWkN<#bcMYl)!n%o5&#+x5yk~Q^ekp1o$X^cDKNcSjWK9tvy zUEsa+VSE`Ob}6EGs~LIwzfk;Amyju@uW{7RSPe17fBma2LPu%gWD@n%Pc@JO#2^l- zv?<}JDLwwG!gBLUUA|BRQ~M#w6h}%pzJ+%b_z9$=o*4BzA;C|W4*jH;%(`Kw3O=44 zuTo(%s_=V%-vtSbhNvXXhMaStBa8FJoWA3AWwwou_){j~;NLiTVa6W$cqR;xr4&q6 zw=1hA)oM$wawLW`oKn8~wDD^%QXoBg+0{?|E?p}EdUTotlebLg1r{?j7I0Dq*@V+W zmDP^FQ$HR(4URt|dcIc_P`fKB8ugTj>9*$On84r%7{bU$N#)$gfcHlXjNu(%B7Zp; zpj$A;(S#We5@v(o=v2$hQ;0)Ok$N4?j_Uwc!eUphAu&)cS{cjjrFCje{q|_RZ7{1Z ztyw)kJp1`*-!ZYX^k3h(HS;g;^@N88CiM>IbC2W4muuDzU&#R8EjzlpavI&MU|*7* zqf-ww+MlN%4uqjt4n0Cc(|UeF;K4ZV*_Tqh)Z4P^vgUfRmLSfNFKYJAkT}|Oj%lxS zgK4Q%w4H7#J+QXp5n+G^BoRu36^$vdEi>yD2(bLKv)u#7WQe2eMk8h81YbcJiovGR zM+-laM9Q-zykMuJNHDLl%B4?Tic77SG8Fkpa|vtZC6v_k3rzTK8 zW{Zm)wK4tm{C^wy?^3cQJr;+pWQ=P0Z*zO|^_!ghx3#&kxs?B&&>gWm{_fVXaRptgaTm=8R3GCp?&g=oAk|M?k)yh|Jf=(FxA z8w!rFCr>Ly|2Sb=y}?m%M1ePkv6r^8W|h-LR3fgdm@Xw>a0>snW1pkCs*F}wS(E=_ zahMfZ$$-ds@j{h+iz&v+&JzQ5H|TC1tVm<<7I?E!Caa6@pEx`>=ngzS{@O8x5yvE% z_5L_`Esd-P>7+X!GWTuMhfE5QM4W=iBFpCqrVj$Yp0?oR2<7Opa zG{v}UugKKjj=eHTLT^o-%Kjt?L6_bP!2fgr=ysI}Lsz*ngV!7C!x(-RFZCPi;8)X- zrPsC*ZI?2WLd$a9pMbX~-tG!(H08?M=TfAnddr_FW*t6D&bD)vv-wWpQuRcxRHI>) zdsX@W$gf#1x^&~_ja1yFo%|rm7`@n)n)d}M_RD(_UyV*w>Bh|)>CCfEPS1{h-n%&L zUVQk+;fXmPcIlE%cnkY--E`%(bbb;1jmuyZv9}*D-gUqCj0I?I+La(a<-2_ME$}?0 zi7|Imz`v|Y&djx~z4Tp5P~=gtmZvZ&R?xqAyFxjUZFRGZhUu*+=ORYc>`#VyzEf2d zFT=2WD^ZfuG)t$WymlSN*o@_F%$1-Q;PJ7q{TKLG69WC7qc*FTzU40!0NuyOO+A$E z<6~c+DF&SGK^n1iY(y8B(z5aofT94r?037M_Z{eV`~L=L-^gI z{_v%1wH3h%v5zXCavWN&LM7m@+~0fmp3L|iNj2M}xNp#We+Fjbqpd(U3dB3R=L?F{ z_IHjnSzoPiwJ6jtdodIew&zx$v2)C2lW{pBQ@*DL)I=)F{n?d#k0sn^@veE}GsyZm z0+I>;X(|X#JyafF)AVPvlSP+TMvL0-*t~RB(G?61A*Dll8R!t}BpDtBmX@-rvJ5`& zI-{G^E)7?nwU`E(ZwQbIJfDrn3?tvFjgMkrFUt-y=_A~*MN1`bMVA+|*sv8lntj?_ zKCXPj*6eCobMj*ONO?w{%P2UFTXohp3<0(`vYyl#wwscyitzQVrmk{DX@Aq|7SnZ9 z(2+R`7xE4PMO6(dR3srqa-gj3Ox>URN&rRE-jDAt&ky%Mo*iBM^YY;6eD8I8JCL>N_GNpgd*|oBd^kI}ynJ_beAudr5E7&f z$VAu~IB!87!1e1L6#3u*{A)rOGE@atVQmeVHLCAkiVde_%No~wj-#Q1-Zisd$!kJ$ zf98mw%$ILl-aI~b(k$`2TK#2JQ0K@>Iq1c8^`+I`%SSz}?tcv1)HZ!o-~ViFZ@k{g z-~VjA**YGE%62Hpw z+*1g_q)#Z91g0v`rKrnapuqt7Y!`SZI{F!ah#bOQ{k{2{D4rf57x ziSQRl5b*x@W^Z$+yWxS}z7b)j_m9anQmlUON6gfpqMd8RV9u6;&|Qg~jwvv8Dhs)0 zU1+?gc)TkVvI86<&c41K^9}8f(5@C|`rma9KgZg@!HkX|2wksJD)vZ|kZ$%>ESlcF zs3!QQ^hyJc)9`KJ0kC8e`QY`o_!E!MCxZe047{#aR8053>tr4lItAq<4DSG+IuUCW zz&I9g_d35I*|sRpcn?BR1g?<}Clm>%KhtSa+K*xssvMuf`(Lk8Ou?%1HBz>wI2sD) ziM3wmXaEvv=OPem^(TN=(3T=LGgETV6a6v8rz%qVy$Qb48;vel z5wP#RZ9btHj42jmBY9IG38(e4C2$#m?EX9Ta&5^@n=h}@E0}E28MDcEi1svsG>t!$ zZWQTwS~07ll;@!JawsO2<=>i6Jczg=&_}@gUmox-y;-+VeuJlBlT_@lQqYSb!JJFe zF4j{iEOYds#z4KKt;kqG=7LBq)72jZTS{jG?t8cTQC}d+<`}t{@;U@~UB-Kx;C zx@pYgz3T)%H3zLQnRQgtCbRa4Pl{Bk)ylKN(k^-(+Kn$)W&PTc5o9&Wnoir|)?Gf7 z?%d1pm=Nh}E3&q~6tSzdLQLDyNUtT%ZevDMb83;NJtB;A#oE$_b|Q3}8C>&ff~-!kk@zoFUyTaN5|e=q)!b#W_Wv z8H%Je!D1)zC%#Cy02+c*5`aAwB^@Od(>jV_*_75G$q2&UpKkFX0ALQY^t(byRcVeD ztBODdm=!TG2@Wt#CgPs%heiU(=&b7ds<`(`9|tt}p7$VL};aU1-dsw#hc zyuBmI7%~;&^>h-dKsK}oj-q%XLa(d_*yo2oAwnAQ@3pqe*Yf1%}r)1iINMA|UnVy?7r7WZNWoVt-5fZ`fWacg-1(~`6 zSM2Wy>R2%jB~o5&NXRf0gOVZ8F4cl878HT^n-vUUqk|nfJy`DlX3IX5n?rkdH=i!-#(W(KoP_2?y|hXBof!Emzrkqx+8d*pa4_% z%&C+QVtV;Xxa0u3i|RO-m*Skf6)6!K}QcIxY@;W49NvNAI z)#Vj<5MKXc`*kgw=#-E;=|gTWtEoAPaGmF8*JTcVxo+yTij<*ey)oKQ-TRXyK@oco zV^QCLrchL!!z5xjk|&0UZk#Wt023ooo~8;k;T{(&bMLfHiZ{-V<~vDG76VSv;GLW21bF%H^reo~%K^GZX;4EWWN#M! zNrag%1_%T=q71>Hmqu~1qw5*DCxw@TP1|Qrfrj7QP6o^pNm*1p)CY4ywAMW zdo>XYtk#Vrc>KGW6R%yXrn7oPRc4IG$4>gg!h%(#3cesx>Bm%CDi_%tBQE6a1H#R$ zkjzg9AuNt_Rov2-2d8rRqg%!oM`q4qDpikp?;0UP&_a{B&p5(tPyD&&ZpKjPl(uD- z;SC;Sze`$8l{2MnW+kk=$Wj4fj+(7X;PgbsK{SUs2PFt*bSTO`KSRF;C+)Wcci!LH^YzMwRU zY#B*<4O|Yyb_7}Bw<=a>`m1}E{%M6sw7Tx4p=)!Pbe0nVY(dJDcR}8pxCu_u@VY{UoCfQuiV>jK&R_ouRAZp~8T1kJv(nD6MIQLoO&y8@ zSv-kI)4(?hcvJ)kMN^aPMX4m9f+A$3L;FIdaV(6l_pX30v%r$(GA{z9VKR)_EYNh` zbdxR>LuaiCr&*2F%?(f$uBmP|pY#xH0sxApeAt=auv!u(F1qR>D^2+>pR$|WmdB91 z_;Q~@4Yp44pm7)c1`fac!nXt)GfAa$X{XL4jv?{S=})taP8BbU=wHqyT|Al>AzRsp zd7a@fmo=#0m8~rDnAh+%6Ef`5t&a43MC2N9Pw_uITaJ4~NIDk(UfeW=pm4;fz#bmj4;1@m&jjU;eyz z{P8dk!Tr5DihHG}Xo`A6F{C>W$?zCWQCJF<{$7OX!;hE8hd&o_a(`b+8`}3)I&<$n z93LE>73RY_??l`gib;f{VL!e0&NXy)c>dwz*?t9L-LPt=(T3GI!;+MLjPSoE=p7D` za8oD_>Ep@Ke}6m_(OXXT-XESj7hBy9t6E|d!Nea~UU4=AqTACwCo(wAPa{ni_`mkB z+~D{kq(7qs5AKSPod4tK^z!2P{PO3+v!i$aoO#t(g5W1OW{$r1&yG$nX6UlC!w^qV zgeW~t$aRV|>*b>OQ{5vcqqxwe{dVzvE?BNA?deH*{z`?9=A62+R6WmKNrUQEs9;<3 z3~O#U!ba3Fsu?)XIo0tqF``C$OKx<@jh3#)FQg$U0p{ybk(91wYoq>Qm}_%XjXCT> zrLTw0wm^FxUc0)>#WCDQKXdckCd^+c<6W}_XXm{=rTJQI(|pFOugxek%*UYDpzKf1 zLmD$T8zX7$t!B*K%vKRIv2|%%OQvqg)RnBl=hQM9Yx^tWg*YoTD<_nVs4*XutMc`5 zK?Y#Y!vj@yxHt|d?PYHMCxiDZ<$h|`+akChi>J|-fZqyVv(x}Cwx=)2y`{@~Hr~yk z3aa1IIA}RjTmfEQU(nN=q1FQ0pvalQVd~(nzLYt@KR$Ny@3ft>JD`~5k8ipDOK@H8 z|7Fm*^T(CObC-ngACqhJ1N3ihVG@W)2Mmk224d68BFf7B7)LjD4u#_9T(5r4K)Bgm z{;DhWhQnC84adE_%wKZa%gg+79=rU8US!IDkJyMf8e3lX=U`f1_kS~d_wu^`S+4uj zjaHigNM_slcz$?Rla{A0Q&0i;v%~)u?eeL#hmgN#k_M>%KzBJCRKp5&)y#3@%0tAN z&4;EDzm1D8{)prb`Sd@%2}Q|2#BEs`<2uVjW?Tm>u4iNzTT33MC;V!@xePN7o5%^Ui7cI@I zVdSNUjLDIBzwBM?|KypIa$kr>u@57T#|f=&(uSvsNSYA*yCIU57h7v^dW)f4o*$n5 ze0X+warFN1!^Z-_t>CSkIGvF-6>aT{fEr)V{Mhh(srlJIJ~}+Pn00>gI9+PU{=6Ep zZ`nr9kpv{YZ#R5?q(Y~qq9$!&sjFG)YRXpD7q@i#I(-2vtZY(%zjD2k0qJ6NPey|+ zjN$A$UylyTgzULY;VOD;tBZQ}jKvGEsw?$E%WqED%TtHX19lX$zm)|$9ynzsfTxo4cWYRz1OX|sn@cb&J?(ftW@ zbmG)~=~ztJ4;%GT3BmW#7`Xb*b)Kdba*)JX>qrRTri5TROmk zr337@qpMr23v7F3or8cZKvh@M;#{7>?X_o3TB`$TsU%w}$(Bm81sJheyKfw z<_HF7pki(XkE5-ndS|KLS*mxQLA_(_y@e=uOw#{6s+}CC(ps_O=5*#&3{*l@bGXzg z1)L?HQlV3W;@QuR*_1iYGGh%CInOp@?Ugt+C|;^?{=6!jr;7LG*ywRM8RBSW<&CPo zRNX99H_k=%oQfOW>sPL}(O@n{X`@?nscb56G`G5@MZ1g8=cMgFk2a^gyT$2pR3CF| zax|>pl97&v`76@Vq#!NcmFx@ZgqBK}r4nYTgjs+RCM6FGQNd(1`*{>F7SGXI{o>+S z<`v~DAgUzKbz;1u+*7JuDiJ#?13R0_<;iEMfx6{cW~sfZr4q4AMa!R8(V`Z{Y-*O2 zfzOmImg=OpKY|JCofs*;7o;L-rW%<+B~qhs$P^6*OmdWaar`q%sEh&OP>(D^EwUi! zTB}I1=d%KbDTfXOi3ul(<_zRhy8 zcWmwEG(7$Jm)g!gi3WIBzlYPD0TIshPifMB06>q8W( z{iT03IQl6g2^^M?=qXJ4A->l5^(T@St6EpeTE_#hN)oU-f|ScYs)qL)C6TvQcs;-&^41(OVAgz>6JiZ) z$xCxIRm8eiQpAb#8D5l3liWJ*VeFi+)f;q21$Ck%VS8abg0*<- z&4{mBl}_D|a;cGZsTK1}*;1<>sAG+-a%%nAZoIb!JgFU!%M0A!_rLQ;$iLyW{YjE= z`kvJQbVMd$0Im@j!6-nSszRz14B)$d0nw7ocV&T=9Gd}X9^3ZbFPw%&cBTEnvsh5( zT>X)qY%yIZ+53erldS1xi$s@y)+Lex`K1@goPf-^JakOl-0IM;$1oP_W0A|F(6zZr z+LV5mo^9@Rl~zF!7cri5b?L6Z^x~Qijzt~3r;0h46xZ!`vC&LNY_i>?ZRTngi*Vz?oKw~(Q8M9d|b*U&G= zP<}QvFN+&W3qWVk9h3~^a2&I{1Dq^&FuC4ik`z@`28E1Ca7q$(lItf`qw$Y6NtLM~ zdkiK+#Q0~kUYrm{d=l=0i~ZB`UWbHI92*^XnID&R{vR9vu`_=Mlb~$}9}i>)hUuB^ z>GZ?F<^%luSS=z)pV5@tHOX20j3FagS+}Mi}O?9ToCUs zP8YL<8};m-P*KIPA9Fh~5B9=V@4TRwEX5fXp}4|3Ls5Vdl$5iIKfb#>J3Ki!JUcr1 zQCovJEP6(a#k>7}{c7uqe!Ge8xXhqlSU<`tg&WNjYRv(}^6c-+R;xy^zju86!`}Ws zYOIp%ZWLUn>OAHkH5qE#vE@fo2hVx_wx;x|=*M@Lr+XJaonId8UF%Qr-leL5XMwimgvD&o0HipwJS4?Cx4{5MmYHv9n`Q zxBOyK{;3Blf1NZvH1o|_TfU7ETD<{U`J9Lf>F2yPR<>j?Lu2F25!;@{CR9%FM|>sQJaqLQaJ)^Z9JxOec1Rok@6QlipIB9DkV8;sVYwYKVg z>qDc?*rLwRqVf35?37P5#&)(E^>!LF^JAJp$$D>dB-C17)LUJ&m=z;F`xjbzm2qC& zrW=jNm-@zjp|RD3ATG22Xgoe60NZnnu#q8&Mn%Rq6qs$xh@nyxD73;Kb(Fp*fhuIY zcpes99n8HrA-^G?IieF;LrjagR18@g1zm&HmrrqZFFwVt_IW*b%WdXaGD1bU$$~&n zz0Lb*Bs6EL+znXR7!hijx#-^+22up1May(K1q`>tWxkbECM=U3tQOPKb?s`RP{w1K zt)^wG8l8wvHAKhn>%l|-@Mw| z+S-2gmyNCM%{Nh(XTPa;hjoei^RD1-4xlurF(H$f?t)LA_bYe-JP+U~ zz&e5n{?j znR~-xyLekwK7F+p4`$Qmp)Y3yeEKyPgH!(XrK!>KdG2S%^Z%M}l4#hB4X8Q)H+QzT zbLaogtF1T7^Zz-XXF2~}r}__a1nQjZzaASK~7CN7QmSMFidgKmXs`IZ1T*EZkJ)nJz| zY619cpqDM4FT;7XKmW%tc3hfQ@dGvI|Hhl`y#Dv~_N&*+^Zz*>$2r|fGgo3_;XIjB z^SeKsqSj&Z(UNh{*l=P%wttWhP{R+J5{u==gk+}u-|F_KUjMW4`b}>CZ@qf6)c-uo zlau4~JbJsW+`?vkF1~seGGX67q&M$-2Ek}YM5^gdJ*jb8EarpP7J;0t^@12sLuuwG zEX(Jcc-qi^!%Y4+N}_Jl^Pu+r|5aZ9^Xm1hohAK$j_1D0kL3QoR1;GhHQ3qUg3l6% zb4kv5qxmtj((;&DananOTqHfo^ByG%>nqK}-tkr{O6xppyNwEp&lQo?HF);qW%{G+ z!#gsG0zTiV$85O_zLH;Db3})BLGc!_WMW(%D1Y~&^e%!p@0t_g>R;pKp=%ttO2Fp@ z8v36WTNlGHi&QAa23jR%Yr-1n5bm?2L3u<;43$;vcDv@!<*EONep;RXC=SWp7)7ma ze`?SFovl}!`TL)Z&87X{vpkv#7s8uOC^~15p}{0PM{KQV3E*%>TWjKZCkeR83BWj} z{gfRzNLSf6AzM1+Hd)$1;hV{6Pi9^aBY)R+qO>LyP8gv+3~@9xCNvA35Qo@@+M7`S zwD0uQONTBkK{80sPna^2+!?MZY29K0SyhKUki#8A=8ul+FWGdhW5?9JEcvEBAg|{#j{G0s7a_Afw8LjW8WCgFdV~M#WARu1jp8bZ07e;U}hXkPdGS? zkbgrbW2GH*Z@Ii$K_+6NlX1Sk3YYd=cWzRv}Y>Rl-O_kMsB32zXuMws>1s^PTo zVnF3|D0IQrT(?_=E3t<;VJ>cFM^MJa$E3`~yj+ujIQ`qLb{t;l{9IT}{7Z8>q03+!2aR6J9#8$W;s1qoXLk%EJV2Du#zH{d{qN>R!T;yY_VWJc zS)TiQms8&dX^5Js0}AeTN!tkv zERRrXQYLivA*Kv?lu-|OhV1Y0ck&nIzYOzE<3?344rQ=zbArymHKob~)Bk&|Nr0Pz zM!_%!70}G)JfMg4C9bQVS_z=cBcA>5my1P;+hhC}5SSBeYQkX&<{HEzb} zmP0rCF}}eC0ns=;eWAhq(~mRGk{?gLwka|O^M&Te1wqN=R7}k^8o?>n(X92aT{GBI zRjn$f_)pAXw&DNmtTElj<^a|F-{$L$S9$;cS8p~pm;B#zJkkkD@0`+nL^aNr3%$Y5 zWhZ(P>DwM?yX$>d-Njt?+buui)qk$zN<#pu-o~9wg#1#{p~LIs|WaWm!cK@Qh$VkNr)2r+SYn4J=}Zcd7bKeDZ~#k zx+;8IM1TGy!R&6IL=1hlgV&oMW{%g|0EQ8vY%j!+B4av@mrvPuoM0k`8A3{JbQeC= zggdY8OZ@7^lQ1l#kP@FxvoH3L)%Bpcj`OKA@(D125qzu!+SDe&$_tPwh%F<9B1d5M zvyz}8rYyPRyh%#v=pAKfe5q_8E_E5th{j)qsI<+lF{*-m;;KnU5Lk7@Yq?o6P#+&l zk2MWIOekl`?<%t|%EAl|!Qf# zV8oFuvS!h#M&3*5yO;si2KDenSirqMyf{1BKX=0`WAN1NsB$lzGEvZ#EYpUrB4wrl zVLowY22XAJ%-7CMGWWpd8;>kEb$L|PfLU=^r@QjJj6(udToNefRyMa9Kqs>i$cWzg zO*7_iWybnG8^<}6NX+FFMQj`oXe$a@p0&$4O(prevc~{g7f)Z6)uA%8uaW3Sy)kn> zduwIY69j_~QK$|p{i_l6Z-}2__5~+$tO)F7C^676<4A0TIdQef!T)S5(We}I>p{&m z^EG6lkiJrPm`h@rH8u+cr~#swS&~0i0?Q{9napgIvVxhD(&)81J-{3g6!Jgv$ILT1 z&*+ULk@i}yA*qEt4qyRPwkd@zOD3YB7$9z|v3P(Q865@P8ESKcVX@Z$T?wle>t_(UhiMuaJ^HSfvkqD;Pv7HXa z(dA+moR#>b3DEadd3Y3BtFrN^p*P6M<3MjdGmm`o`FV83sw_RuW2$5p=a{K{!}Yw;({7>=BasS`g9JJ2<^VN1f z{?E?VGXKx>Jom-;k8UG?##hDQ>oNRXuqDB4cA&1yi$9uKsMCOR_E{=3;4EKzjmDVa#MHCh0Lz$;g&G#l2vSos(7NWoVWZF%L;EUba(08!@|K&Mk2YxIk7G1n=y%IIwYp1eWjEB$E#a*nm)m?CuVq9TFf5@3GberDfml+$l3=Ia> zy>d63^x~~V3HJ4q{avn_5w-)%};uR?wyOEYTApMSjYT3Ei)@;zN}e?uBhs^ z$RZFzqp6mOMhvw~tc>HhffeO_E>7iQDcRxILCDZ^(J^F`F-HE3MC?#w8yO_*1%P2l zZb1-H!R3dOl;JIlaWmu(Pc85XKRecu{DPka|?@Lqmt{qOV7p`#gaO+r8quS-5Qol*~?1r zjs}r=n)6pX-=2p+JK(21d^iK41};xY}bl;q**!2Td^e;m4AL7MGtFcRzi5*~)M3X~q9Z#eM4-kTv|@ ztBuz?`TVcj8_W2Q&+_D%C^f~J)zh&aSJa2S7$4Dd#y^7Hw{9~Q-3C?#r6G%OOD(i_ zK8b}UK-~#tfe`8%rzI7ebao2a=;AJJ-ifZ@Y@D;H8sthIHm#vZuuvcvj;WTmA`iQ; zjyJtn6^MdCa23`AbX5@F(n{h|j#r_s^2XPE)F;<@q^p%!cj_x*rv#9payE8)!D*cI zP!_7m)97iWyDM>I&bpEg$|~(67h_bZm5St!P7dBs1J`cN{8(_lfaoOu*5+F)t7`bV z)i`7+$6_%m`9uZPp_GzIC|Rr$LO3FnQIJ*4`RiDvQtLqe(~OQ%bPOLcX84luJ!#!t zrDHZXXTin8br>?nlVHqCq$w+cmRz2e>8)}vFx0+ z8Odsx27YC-#8!ztLX`tf#p0_n3U6t)SUZElT=VcVLrDY;i;KLG_-r|=2!Cg>Y2hS_OgjoDx}ot)*Y*l0*@T_%tT3DK^ImsGRNmdxivikmNEXLyH>q12pYZ7LW-8 zw=juh*%$|)`1GeR>HE+hA-OB$|9deG{??IQOi{!rxYraR z&WN!%t)UU2`a{fDJ&o~LCR(C@Q$<5Eyd02Z44EX>I07#}tuQ=Bmm%?C2t zRf?1KToGRC`&==KB1f@TcA@LZV1PJrvL=BH26izqA(!Gxq-w6j-ylR&6wVz1`dxrsx>M7)Vr2;$Z><1Xi*M zPT!?#AdMQRzWgoy@bV?_LqgLxkGcoW1fj%eA{23(vcY}7ks%ZS=<@}it!#mz^y~0< z99#H{{$_o<9OLLRMA49qz~)Wvn@`5^rN9~3%6$@xQT}G=Gwa78UxF|e!4xn>*f6H@v8;89VB(KvXxXShC)bw)95R%s$+ISlnwYW*lgLNrwyOX$ z#8cFFKqm~h3WHux~GCO*cZcGnZ%SO2#!@F@-V7vL)7i=%MO7TPp(7kduFYcTcBUI2eM_I zyE>DRI?`;lA-RTO&xeeKnae`}JkgDpOBIl{4_vy<@PQ1c54Y%gytcAt(HQ{H=NKpYWLD3A z*hIk^OoDC`%nH)?>%~cPXNOOHUj5KPEwdX)=lV zsdhaHhNv#{pL}ZM|IMxKeEz?kH(N{p&u4jZ>f4wE2k8|5Af`Bf`{UAoI;xdffSneL z{^{GAr3QJ3-e8>gJrX};pdML%(ZU4D{X%^^Z44kou#Jtd7qZ_ zU#A>tWdl&>|G&AJzyIIa*<8~9XL%g{|H(D5DWhfBMF!IQ?5@Gf001Ie_EaYQ@^$E*Iyd>|8M2a|IMAvrT^EnJdX1}rMyO5A1hn9 zRlrOPvIbTSb*Wj5-Ngua@znR$%0dL4x(NQaDjM1-VhKE*`o{}2D>U0TO$_I?F|6ro zfFl^<-%+A_%3jQ9%qVXx-;;nJTf!af670TBnHu2n6Ar`)nc@He$Uul7Wq?FTRRv=b zATS__k<~^sEMP)W0Iu)Ak8%<{H~umZG-LL(K&%dMLfzZRfMv~G3AWo&1@s_GG+X5Y zmP;9@G}0;OKp9Cy@?S$qZgxuX@40N31|3;-)d&f>nZ$BupTIF%1FP~C@TQv{@SMBf zbNFVOTX)Uz-5>>P3Wd|OS23XH=(xw%cQ5O>?M71*kaLX7zipxXG26CuFpac1)sYsK zhI-%b(}w>`NZ26SWAUle|Lkn$?f*7kE%X08&r_!V@e>3YviUyoVM5WN;`z=_*l5)jcVOG-71z^>vNO6FCnDjEkqe9rutO$eZ z=r1i3hm#?Wir)nYf}7=Y%}x|}RKp(xjm49Ek(r~0M(Yl)-8*DqTN_BqKH zDkGmH2<)96NwrZPub(_$XON}dI`E`wk9rnTKY2sM_}5#`S@1tXOqu7zk+t6(OD-$v z*k)VSHq#7=@&%U56ny%%v*cLx)Aszo=4&HrW&U4#|G!;`|Gd4q)c-usJWXROaM;l~^aN{E@TzlhUKzeDF}=L#eIZz7!#Oe4Hk0e$kk9CD zrwgpmo_oGeCL-})dBfBydNRkf|H+B=0-rYYUmopU6iwSnfi?7hE8MIFCWehE-&959Ur!;Qd9Wg z=zQ;oEx|}Rbj2^EU2%{ z8zh37IrUSjhp@dXZFr9f41RzijC_=MCQRmsDzHMMoQBPQ*ZlU})uy8nsr5(^9HAPs%iq zIdDt4rj*c}kU+$A^_1jMLL4JiY;A2V>p{PC4px7xfzTSexam6l91|+4<}=$iIw1-@ zHAo+~Oc?iPeP-bQq14|u7XxbbKZX2n+pl*vmi+&7JnlnQUD1@>1m<~uefI+=jEwnT z_WKKRgCZA+)V=U}@A+TgAq|mKQM9ZJ4yGx6JdK2)A$bBpAsyu0@`8gKSYFF5AICET z{jX2>nR{yWKU?|u58JQbEbTv@<#E&huOj1TsR-CQspnUf#&f1qry^Bn?J?j~8r7xj zY@DR?Nz!tDkwl#BQg60Pw)xcQHOGi0*gwnX$gJK!g-R}NrfJnCDL_qF%0qxDjfJ!@ z)zkLrQiLo;lwMx`YH1r(R!`N!CDyT|6BLKg7yjDT#MN}%9%yrW%nUZCDzgBd-|d7VA2>sOJ;b(y+#`7HWr#s4$N&|nhMmPSA|{Qs-1onrj= zjivtQSsrn_OPNUh8Dh$SM;Y~icdq9895Ii-OJ_dIe5&spHF=s)WYo%;;>;CPU<7G- zQ~?hC+FCxW0E}a5NFy0s-^2y=Q(Ez834pU1*Z{6TQNM@wt(ku5x;a?jS((qnS7$Oh zLNEwrmLW}!b~V?KP9}fCT#}O30DhlcqPX0sNnl5DsK9)oCF|;^Fn|28VZe63jH2R*qGE@dDGO}kh^A5&cB66B4gd=AR)8bs z3dqG>DZ!fvy)w^4i3Z9|5b7d@2rvs-!b&oPmFzDwOLfscX(^CCz)u$! zr~P}y{B1qHTvNS$0^SPm(F1Sa0&i<$!vj*`y(Hyd^D`s;Zxsr>j{fg#=KX(PzuH;) ze>}@$3c&M6=BYkL%0dD?UvzMN%BjB`-hwToRZGNo` zmTeSt#^pA?=VFhip~uy4BXqcWqeGFapJ^a+-2lbtkP=Iyxe`~7{ z|L4ud&XWE=$MY4^|ECNf_$QJ%(x7~A%<4|yQ^nd$R zp8mhu+F8c`c$VkerT_B;n)@>y19QcpYlqM*IvhWBO5(${b49s8lzEwD@6sVW$c#R+lqDlHoao0 zQV7@D255;$kotD0)5Lf)e*hRb7~lvq73)R+DslMMAwOF7!AOV_(+G{lr8cXFfWg96w~T+Xbn0UjUwhIy8|XiO+S z?L;Won!*eYmFG|jJ6~Tq_KsIwJu8~E;gj(ga$@<(>yw!EuOXxTYaEF`ymjDpMXCgK z+)9%HPdksv>kttw!LYRI0udD9RJb(CiiQ=IoR$1ro!QfHiL9k2siv^fvaQ~#(ppBj z8=fMx>77s3McZ>FLHw9!de%h-EbsrH z<#_>4A!8_sC}0H0X$H3=6oKmr4n^AF82UGGh-k0#0$hwR6?eNLIu#wEFa-RBmj<9X z8m^8)+~6HF!)9333ePH#$m5}-~`-7R-| zI6$y3%O~Ww&WB%44$scN38GTJ|IZHh4&ER3#=*>eRPO)H*E^d#Ir(q%&DPTX=XstN z+E^U?gu*cp9u=L=ciaPi7dc~80fcO((v?6rkmtqJC6*_~u zLeT{QSbmU6viMs`%0!xV$^HsH1ze5qy6VN%ZV?B%nXHt?hQ@*%-isIDV}#3qndnb_ zDaVS>Mw0=GLvkmA3KT(5RkGr;#|x4DOJXKLV=_e)=mGyDbr zfz=S-ATZtRZS}T6_wQi3)!XQ8tXa9uAmgAU2w)0e)WH!S31s*>MBq9>@PJG^FQvbxz&(JW6^HCP0OMZ4#N&# z9fE<1C_^ydHOqz~7QD2apGe8O(|G`=DMIR=2XKI>pWs-qQS}cmFU9A!&qL>-+x%%$ zc9!x+&CUb3(w@p!_2WO8jIU9`cSR%|yXwkcZPqqghA(gxli&c;WFn^Lhe*tgNAV|xCF^=$fG6ql=Ex;Awa2CkQ;eww(Mlc6i5ia;yz%_}SkUarh znQHmU>?a3TR?6M_d~jvwr2Cs9DXUGREi9eB#KBzwEf-wq%SCkeV3C5Vnl7PHA2an7tr!`u7sJywnWZ+XXvsw$_1sBkVQO zR|E4mUn+73C76gVwA*PJXb(VWW61!VswzN6vFDryrFa;Dm;}@WY@QvIKl3XPcEpQ< zRE1mDI=Bjme}j@9CQYHUAkymJ(#jb`siwFNq8!UU16&La>uNoK4^xyRI8aQqmfb`H zfWaZS9TCCo%M%|{lLL=qO)pokgirA`;#{8KJFnLGAZ9!F%t6pm?a(!H=?H>VB4{}b z*Q&}d0IoPy#Me$bFRGkSYQ}MOG&mvbG(i+a6$GIUE_ESQzHJS-RAwcx>cfc7pT0o} z)=aoQ{W=rI6rHnDz*NyWX~37t?y6>a`74%8D$lGYaAgS{N)Kd1Hq|g4oq~bR15t!) z5%{w6SvcCOgzK)j;vT@2=gkin=QJu?al$1+E0vYMN)yAh1J8nsAF|Ei61gXCg|Mzc z#1tTrDj`VW`hH`6eCdd%6?8%$qnx%1z(^$ic5i^J?Y6*Ka2csD-0eJoy&#YT-^`sc zz$g^5X&%Bk>1r{lN9le%ie%mrbLs@%O3Ghx&TC#eq4RH4%%fx7r&&9kaLvOGIkG{O z&7-oKGc8&w2tMZ@E9Hh}a7o=!C7ZTCp^S{RmZ`S&#lYp)(V7PtWSFlnRo0}gPPi5^ zk9IOCC%r#91uFWh!D2Ha@;QNPRTr${fxc4XvpNHKrErQNW@Nf{X9dNZhCT)HqOO{l&?)9|kq6PvTYMVvzIj*EiS5`kt1(|^HC$iY*`wi#Asf~3s42dDAH@s>((IuEuKpa# zWfQK8D!x90OEB_ePz@G`>r~idvQ1KzK2kK8gwmEqf5Ooa zeDdW=RyT<9`fTr)bFg>(L)zg!iC99y5GC0yb`}z`OYx8zSL5)OcKNa2)jWj=lql=> zr(3<({ow=$sINxeH}-b_g@R+P2hQ~UUw%Frk++B>LVzS-OcErAj3N;(sJAG($tsC} zH6<6v=QhSm$}a>i^>4X`HG@lPRC%Euc)#~g&a$Uhj{GPzWDa%jO2OaXg3aFM_l+ce zr6X5Hm-V`~{nL;A_h?L#yVO)O4L#To@fceN;GA^RG>ERDnPLlP(sS?J9R z<>!LSuyU?jUl(c7CeY2fzO<2dgGz555GwJBl#dPZ_pyegQLsuhUiw zir3PrK@R7l;Hra3_H$lfIjcaMQ`ct0^|Lf+b~X0^)JJK3qGWnuyo~~YBL}BE;~LB0 zI<8}`9Q+kuTf!y@MtFd7O!5L`dw+z2Nhs5oRPzg_uI=V~3eu`zRoa}e5sb80Hk0&h zz~uHUrNF8!VQ*1ekJ^{Q{E+M|p#X z2k;XP5IFg8ad-}HWs!9-F(hfxlpxe%sdDms4QYGaDe(@o{Smp14j_Y72fmno(f&FJ zNc56%SqiDsf!uO7ZG>Y084zw}lNdY*ueurWWy4irNAm!9(XP+Sb_Fmg->7UtY@3={ zEN2=(=CG$}0-0%B^8l`{C-m-`wAMhC!sWE5sfUZfp;i8Bv$wt3Yi|--3YXJFrXDWa zTxL#z#RZqsM5Z1t+gxT|qZkXW3N=R&E-uFvG#ucsKxcOEV$M1zEk z0nAV-*Fg`QBY6+EcX|W@;!{8(@ad5ErY<{O`cjcXhWNCfpaDwwG1MQT2qm0rFx9Fs z#{KDLZ}Uz61&XFqn8;1i3(GUqyS%ZTzIy<|A1moN%0S%Uo=+J7;jI2v}@2z6N;b_bJ4 zNHl9}P3esqJO^Fflpo{hrt13UH{>&inGfZ|b7G`7i>%=?twgr+X zX-`oNLYle9#MeA_xmx!vTdi1PRqHDa@sV3;C6)&@Q*79-s9^9_E58aZ9ZWt4){IL` z3d7-Ki0kg5BwVMms5&e{0{LkHLlO<;t+Cz=y8K-0o-@&nZWSL=@5O9Y8I=pi1&hPA z+4xDhA48O|qN<=7T)OfCfy(YW86M!uTc~ol(#p>AbFtp3qyDK%f?0k(y5><;*7@cV zx^TGa5cU8L(xOULR%$D}P`GLs!v~WEOeG0*ktZ2EnjvrwiBrOr20n{wu0b9&9e7DW zoZvA`?kX)K=7r1U)m#FXC~E2nQgxKRU=-hIPr(&U!bY-i0g%%wHyRfp<$g3huos5p zmI8!@Mi)i^CJDUb;=sL(JC!-@Jc6+h2yVWoUCQo4w6+e2U5BznT-UD(vl(B;oDsn+ zWQE?`l-tY?TPth}&fxf9?-alZ8=;6{ztntY&Ra`IcG|KN_+q4DWpDx5xQ$c%upvK* zr~;y8M#kZl2-~v`*a%Y~q%s*gVT8a9x|0RFqF~PwXhj2ZlQiF40&6ukM|0}sY7I2A zeheqmvRf`2u6Z_nt%0T!t_H@M^Ij}jseLOI6Q?9-f~bYT)x_*V!`0BLbs=z_*7-N7 zeoVuQOn%9nowP|$3Rg?dR|~EX50HQ7hv+9lN(FkLR5Pk581b^&4CHD7l#y81qH7vKb6LztF|q10NFzvFKDjP{T;;F+HGy~47eY=DO1-FyAXIx%KY*(n ze2a$(4*KFCm9^W!Q{g?GhsVC*al*6N+j`w=?}FxltHL2V4_BQ-^t=wCCS0c#{?K{2 z>Kvlyyz|j;)f%;CaAk2wD_dX8ma7f?kipd;+QZ`Dk{sX12>)w>-r-QhP_l6M0M0p! zE0`h2{sJFQj{f`O;pMxdsob(d=67KfqIeuIesMQ~)& zpkMmy1``pfhbg1D8+1n{g9ok=yG1B6seo(zs#i)k1bGUy72AH0&;@#BJVH5h|gd zn;EtS6+~t|(}`0~V6F)piT?N>vmiJ@A%VfivtyeWzvIW~BndfF zo#{(+0;Q}guD4MX!~{odT?Uq$k{blzfRiF3S7TqRM*Ff$&y84mgX1$%z}ozRKuU2OQ$U1;g`HB~~jJJ7x3(MR(MULr$cHxkzfE-;;ri zF%@AQQ6Mg2(vga=ImraAr@q=Sy3?&nmOn6Uy%-*&Ud%inGpu>wI79sIQ!G>7*M%ji&IT_0nef`QpkH;nIDG+#Oe0 zKa$r<6;+mGTmTDXZM#M}7hEJ6;&eJ6z)_&VA;}O%n95WoeEDVvk};9$L^e<2M)qyB za1BV94f6pUlOgtD2;}SReAGxAiln4Do%swRPD8jpN}N4t1@9f1m`AQSB^YGb2Op0P zQoGpD9t6B5Q#1?j;(|*V!ybrzbj2a_8J==hnGpb|1P3bUK#24{khdWy$YHI$@oN=a zA|Av?ETa)k1*WBECSFc)&<3UB6L)pAktYZo&Vrz=a6pci&u{-yX(+0@5RUr6II+}T-*^W z7DuL-!w4WMlB7CeD8rFn-vUv>))g!Ed=8(=W++PsqXXBr|2fv|{YgJ$p zi34DSDi?#e7nm?}PUS*KP#Uwl;V{bv-n-}-*-ZdrRgsHqT z90HMjMiv*LI6fnvf4601E+P*JQ-ZbHdGo;MlvZq@{Fe`B2kQ&b09a^uXn3Aq#bZjV`VV)Ol;ZIZrRp{Z_I2pHZHT@F zrMCfb>9J^R!!M}(HsQb#W(Rrm>?#eof$0h@xY%&&u#)c7{yYkDlf<;ulBf32Wp<3s zPPNRedS+d98C-7ByyjHP;AwEC>B6nHU=F~dT;AkFYXoN7^DKkREg8*so@F4Lr`ame zE>v{wk9V?V7VFRuPb2}Sc_S{Oi>SkCjIm@}XSMBus z0QU6vYF}R{3}pjt`G{$uD5ipaC{Ttfvw9cjBp<*diKQ6eW{@s0drUKLF z2VBStjj`A~of>LD4obs6hQI3K56r?SD-ktWgw zH7K1Wc?|_nC|SWPb+y9R^ObwuaTqQTb=x?X!z*ni5M|E*xL}$E?8!CsD^{clyH1)A z1p?f&ANKYuY{=(an*y$s)vrtfAmO6mjD!+8{>L?rM5bYVFQrU-^u?IUy_8mwzxmG} zWJQ{qu4oxoWQLA>1oC7@fI6Y*P|;f6O34AISTYC_BF($?+^2M!e*j&XZGDIp6)6+L_v6(#MCuLw?6dyZ8Q)A+DH<$_uTrzAL6cKRx9$Epx* ztOd2skry%1Jq0mJ#+b@r-H?J?6iV-Lx50%4*GG!>;&?9^mqogo z0BcY+S0c;L6_Jb|1MdzsLCsnrLs*I^9%6133 z@Kw%}u9qu4V?@OYsGRW>OQF)G^Ke~5M*IAK3xca8o?;1H3=Um3YY*URy4l;AHJ+ja zF1IIJDO`p#ThA^(j}My-SB1xz1FlM^v3Y}AklQFEFgP%c!CfH(%i{1!B9?^-6NWar z)vjs0lo7RRm@D9Ngv$H6`xNTkZ}^$z+-sk z)XU`%<%pddqH+gfend}_P{tAI^FJ2?SAp2M;Hq*YegIc*dT+L8bs{c>?-!7suD?)foyAr0D-=@7=rG zwv|Qk^EW>Qw$$F(-6m!El~lhZzvCup>$~&tvD4kNS8k6Dk&wiiBv=BJttL6^evSL} z?kBkmGXMxu)WeSBw0oOtwXsM%1_NL)GnhwE-D54uSxp)Wfmb|J>vsu1gqa9*@f3@r z5qMtkR541WAUtOURUbaZ+`A2~ta&--z-0%Fw*n0iV}asjk|5Q+9fgyD3o?67Y3s%7 zkNlF1;Cd>P$&4IVXwtWsA}px&3;00J`^!VoUI(tliQztwV`HD_cE7lFa^6dsEaLE* zd|E49BAl>xmD2}u3g29njqVpARcB%yu8EXO!VF#uE}7`)S}p5qK9kol;o*hVVwkHl z3)d|u=nP!T6o)UXR5*^(*R1=BC2|(tn`(ce?sXkl+vicOowsqycqtp4gnp~951kup zTyL>Z7&A$+31m~g?$TNamy*qfEQzx@;Zt!q@D1m=o>M|VAQnM}Y{Uy@*IQs9Ts!(z z8%(&0Pg)>@`|#@98j!`=m$}0JG9ZTo*zE=3#P4XUme?wQcIfc>q4>^rbAD z`cqoqiPnJpCSa~Q+SiIG-uMgjCfjPI~SA%&i06zuH`aNwudf*%tbQp#Cdh3IE z17f`_QrB`Y*9qIz4(1K0;F6$~b5J&Z=CvS}pB2hwIrf(!^G7Zu(G}MO(1Lk`rI{Bo zhq-IXqqq!AQ;&p@u=!eGK6r4`9kd|1++wc#I*fJ`%vINAOTf%qf`rat)4&E%K^Lbu zS_S4RHNy%p=gH>Wo+%_=Zw_bf3b-bikG3Bzy|h{arliG~$>SQZ%du&|z2@}a3{0Bm z#|4)>|g8k8oBJ*Hs8R)bvf8>WMF-SSI9>PY?Wo{7+ z$3&B4NW%-T-d+yo4R-pn5W630v4h&yX=`xZ6{xz=7FwpP_&_qNpJ)K7gO(7Xbrlzk zC{N^M!_T4KR5r>u^fG0?HdIRhETVMH4TZK~Ja6YmvT8inui={RXT@ z#v^vh5*EsNqp1(%HILXjx%tb3wCw#5SzG6TPtedEX zqh31>Y)DG=<%6bLp^Yl1kj9psVC`$H1^(C&`_@JrzS&(v;1C4$h;FUz*4f2j8|I*a z(e?2Q?M(2jRabf8}xoLU3l(eUDu7SS`F z=2XVRIH^+;JK~`(NJ6)y0;bZHY4?YmM-Q~q6yRsd3IW%9Oaa>u`nx3JOsFknvv$m9 z_amtBZSL6*p4aVmiH`Aa=Nl|}760dX;NWOZ&4XYUeS(L4$~^D<{Cq~!#B-AD3_PN< zeyP6*iFiumY(VsdU!08thMff=PoYS`xjZY2WI#G!Z{(Uc-V}pg@9Pb$zA23Az5_L& zgEO^v&O|vpQ-BV@<;>FqP<`tqr#}#)ByoX2`J~ehk7^!U>r_3SpP$#U#7AUKF$QRH z^-!Ky<%M!=Mvck|ZKdmqE7e3@b2e8>YvBUVGr*|T+LoPAH5T%sU?S&e3SRGRP)*-l zZG|PD=qx>$Jq_>%P#pt)dQ)2Tl6Yv))N{+}9vEkWDlxpo3IN;rmUnJ?JKrMbp@`2RtJ1kub)UoH;b=WP%N2X@Jue9L7PV;o zvR&Ui7%@s@YPovnH593(+U=4f%s%D}(0dEijjI31h+UEtoUYwmWl=4GnHEh0@*NO9 zRZ;Mi5xsrQ4$KppX-6&!^(Oq%I$6+gqWf`{f7Okebd75y6sm*DxDekF@~hz4fPC-_ zRe$eQ6u#qll63~2^CbuZ=U11gGa_MPRd~^SP}9}?R`L_`rWH; z;Cm?*k`*p?8`GtaOY+^}>5GHC+LnLr26WS0u1=K9Gh)+)LU;|u(~t~8&dIXK$p1Wj z^IET_Pjz_v#<`tOoGS%_dJKw9WD}3pViKtRte7*C%>O}oxy0M6Z9cuFJ)PrPt zxX_rhKbA(rM>o?gwS9Gal?8q*^i@ISOQ1=#2T7daY^jPvKo&J6N#e#JX~d(c#V*oF zmaGtjm*`l|GtFRoRI3qa6?G!cmc$GWz-8ss-d3#)~nVuhEniMpy(5pi&#-lY4IBfI`PGXOWJ_6a}7lND3D6G-X+Y@^dT< zyTdc5Cz3N~mbW0{2Ps%ohDPePDS>L?T@|jpdP{1~%A!DnVMS*0g6FJAW+aV^0>)8= zhH^xeDvl$z32NS*u^8!U^fQjgYOaYE(n5O>{(XdDKg*n}#_mzPfp+f<$WKBC ztq_c;m4kh4c_v<)F=KLrlj)!!W={p4r{f^0<6H?ZYFaVck80eFz!qorN(_Vt8K=zI z!LSuzYe!pf6t4(?r$x%wIDyubXZ&xHIDb2w4QY|ot)KqS1G>Vky!i5|cYEf1({eS{q zG`-;l%V`n<*+P&iN>q`DIxU)yx;h{a6r=cnbghVw=l_To1F~?Jd6*3Ox$Zw7`_V8^ zAZyx1hzmMOdQKxvspHj*XijIQ=PS#pw%$@nibcSL*EpTk_YIx@rVN zDwqNe@$AaeLL@1V6xMad5I{Ws+0)k#JxEe8^FwO>8kI@nG?LP*H#y6m@hlSqvQ4^{ zKuq_8y}#B=9z_Y$C3lvT40)ElqE|2D5mN+aK>B-o+sm3Zc&{dT8_IvXz5C?bV0Vjr zyS?}1+h9-qvH#@TU|;=l@Z{UzU<-;q>2Eu&sXj$W6BZ3CWgI?T@_5L3@>HwF)fB?6 zVo@w>``*b}QmHvqDTQ3qDjcVawy3)^BbrMkx~WFrX(XW4>7WN@rpAmR;KdjzT`VNt z8rDf(SF4Aded+5+aq8sQtm%Q=E#mGv*jrxPzG~T4UGf#N)&6Gfik~Y~mMUk8rU&~MkOJ=rI&!w;(TJp1`uG$98dO+`%)X^yzYIQ8R@*2Ah8OENbO=L(# z973jfT@YRzmU7b4V^!E5war>TwZrPvG+1cXJf6o+E5yJ(x{=u?j7F?5Cy4s|tol0F zT1txx&2+*#aktDXdIgeA4$E?^tH$EFtGI=>Qkp+S8!Lf2B@;_2SjW^VN=YAaY4ofRyNvq%Xl_PzFn4`&0?$KzI?X=iFSk|{e&nGkf$TvoXY zPE+TnWvF!TW?GgVuy-0{tUj<;OJSTV8a*Hn{_{Wo z(}+e8M{__PT=A=+c~|C2+6T|N+KE%U1cC_wvkVXQjAz%K$d-w7=W1ySW-`s0Zz}d% z4+WEVx3X*H*OQ}f7%a!4ncIBx52plgnQW+` zmEc=s!Y|oWvE(Mjr_go4F>Agy?t^$9q@`p^@&&dO_LvvsABV4AYWGs-5Y@v2 zy5dgsEnZMSql+#l;N$$o1iTJrOZ9q~{I}xv{iejM9)!p?>3ngnIo@)8yHra{>#TY- z0+E-T2+49)O6~D2(4uDyw~KC}w!QVTF{GqgC8||VZ}zOpN?4C#Vk+CbBTi+P4>HMp zYqeJ>bC=B@_UJE|MoYn6%fnTEPc!hc4LVmpS&z{2Ap$#XGu1lDLJMu)xIU=_ zlGsiyH%Y<9WkQQ&22h~oh@m&U~*tKbP zTAgMP=OmUou{9u!N-`|tB*H5pV>1jAj3TRnYv88#jAPy=mZH7B{HqJ7+c$hOU8*%D z8BP1R7=4(yO6jTXwCu?OMSE2tU3+K!+B@^Lch|4ITd!SXCe3BW54qHgMXS-|o8PF_c`g;gL|CDz4oKr}kUHLKaUo=UyESKqEBo=bJK4iFsMf zJWb)lJYDiL8%{XCAU~bF)Nvq3v5sTbD2qnkot73XRbe8eIU^A-!KI~CyM9=puY}%X zGEc~;%s{LP8f{5-rKF+v@5%Z5_m+l`jvjKLWw#Cq$j6W8fj|{h_Hg3L$Yr4q+9K9DNBMxpe@69spua3;06IFbcQ8dMZ*4yeN#!)KNd+ zhR7(!$}Ik^WLs#P4LP=W#%jI+cxv$}E0hR^XCah^Y0uDTk&j5M7t~hkmID16EmQ|~ z>VK%hYdhzVo)13#;gs*gBcKR>&Ohaa*UN+Qmt9kNvQ3KJ~%f< zj^IVfqJX?m=NOAr)1qrqp&mtikt`E!&Lvk4 z&w{3P#jtVnf{o%UvSHTurso}HgclLU18=XW>#$%PY@wOqpx1MpU`%6AH@7e#e9fd& za>a!i>q$7rf{LYB;(y~9#^V`M4<4jz1`tSMKswnt&aUvNfIp27TTri7Fj)US1_v0B z-dFvVbxOhff{oeLfOHga9nZS`SVHmtZv@|LHob-}9trC$6tStd%<-x6YHCkGW)$UJ z)IU#f>OuElOIk+VE`+cwSTZ9v$dXm5I|g@g7Hge`hE=hE!Y+e%73qptA#SW0uF;2H zx1y5~D>k*d6N431cDBH=x5Peb(3*sf$%YfQ$KXoahwFCHM`{z4<4#f`a|Liz!^mKv zx7s^e%1Hphs^ajBt6#rV%a=Sc>lgh}&tmoJtFtpT-uFEw!=CQ4r)r+n+UeOkAFBp5 z^w(|#OMEm&6t&D);MikR!z3KiEXoOpBhJWu-@m3+lLElk;P?QJk3D+GC%(z)ZzXFO zB-v39$YYrNr_Iqfn-yr5%7jxx!L5e*sHcy@ZUX}5IS z=BMP@=@E$v#qJaEg(#RTVm7VTkm>5(;qPU^$lutE{2f%p$GTmIQh=uj5DF-4M(e{B zx*v+z$F3|v@Tn@dx%QOR8w4hX&EdHyGt+Y$QbCjwu@P7hqDIp}DB?ho^_bgBmAOWU zW-bn;EaG7)nIi;YYX(Kn^x>7x?8uF@lktchm%}6$6CCsj8x0zw9KD$OVZn_1AF$49 zrcHnXa-5W7|0vTl++a1CNjv$jU@4cZ7O*AG#;xb7I< zfn3EYV#-4u5{(iCu!#sXv>_+Rx%sGlyAiYJswd#e&KgKAsVsYy&M z7$o0RB-c3<7qo_>xui1>qz03`Hl=9OYgp+y=p`q72)4j9NubElvCySGW428G!Alcx zTsz;8b0vU7w7ZWV&(HBdlTLz201*^!RXD0(;W3ZymwNinT-Db+t4hd(p5=@cVn8Yw zd^5Dcx7*+DnPLg}$)3?D0>+no_5S$H^RuJlkF`=XnyL;3TUbzW$n8K?mNEZ!|NJo@ z^ua3p>Mo!M#96escDH-EuI^)ddwcs}Zx8<6-rjEhyS=@=^KbpVhr18=cXoDm_y28s zXK!bF=ikWoT_B(ok{18A{mFeb2lpp=-hcek5Zpgs@>#t99mjPv%qn`%)E78Yuhqwj z{omi~?=|-SPJes6lI}~&j`+f09xS4!R zyA!TDZV_*s$m1d&w{1u7g3Z7#bKv=8l2gw&30*wj>B{qMXDN-fC=PX4O5JNsL=>)0@j?XM1z*7TnDLRR(05B#!+PN^DNFc^Hkt zeZdoTTB1Z}&W%P7I*RG?#xYHnZMPyMt+d_ilAnqM+CuL==b)TGSPdueem#GFRxO1>x2VpxVm?Ahz0N(g^ZfY8gL^N z0a=pLL{f1f?0pyWa~}D&e$S*mfLH8FdSV_d&_o^=@ib1@n7t4oP0&8k`VfRPr^aqX zM@}qwJ|OQphc91te(@^B^aKH z&Cwgky2K$L^*B>Gk8JJ?t;KF!musrf2zJ)8QbV4FELUs?G}BWn)vizdu1oWr7Ag45 z+*Xr%QQc;XGZD(f^Xj=z7R?8>+GdIc*ISWq9c{83g5NZP){_C0J#tRjEHc*0imVnT z+ai>hs8BZTk-kquYmg*??}i2&Mjppg8>2h(*!533{?>pT{(MSa{Oyz+9lnC2N1K?{ z!4`Njxw8N+^Yl^#G^M}u3}`I75awpT6s#DRam0E$xS6k;^mUWI>MWQ@Fl215Os`9x z!G0poUpI45Hu6HxA7yZ#<3*WHY-tZbIJ@DxUms(H`0AJmc7r|mjqWkb`I?Eo8w$n8 z*s;c7!xl?tL6jNyngHx?uHYk@2D=TSvr@!0N=X%5ud~x>6k|~)w$g5NfJ+HML&4kfJPJYK%z()Wqc-IaoR0rh8w{MS6 zJpwMBF-@Mcgw9S`$g@Zc$gW~KBqr&!FZb)?RYzj%4A}nu@}OxiT00YJ4`T?q<_3`T z0xsymY6Ig7aiYWtF|n<{jeX9O1Z@El4t&hv@saVyMg|xczNc#;5@3fG#&w|X2Wd}2 z5KPb(*H`2uX5v3erdHX5_h;Z!d3N^XzdQdWc*@R@%{>Fc@Kr}!&~|G_sC`8&#T$^v zj{}pw{pnL%!jM5zK1F{zYbT6vDc-7|A-EcZs(|LaHO!(+Hy%F@PGKBRpT2+mH~{wg z6*OIlr%yj_lHrV~LclJ5H+%YY(}VqGqB%PW;V>9>!TgG{qsc4W2MZQh;RXCtJjaP~ zOXGLTawc$iSQ}uLNd==<#TAB-v8M2!W?BrupK)T$C|t5=sj}t9bw?lU7os|yZTe|rF92>m=u`s$ zSRJI4N9-?8I&qY+&X)PXFKI08Pazpit_f~!&6v;JEb~DPJOx?(O;vvYU+cxKmU(d< zQ>0cR=h5OSIgjR71B`dz$a=LC^ww4nosQ2GZq5|-c9s$LbEXc|I2)e{CeMa5$;4+` zN7n-r){};;4sN?{=B5gsifgWt4WD^73bg6G)`iadykH%2c0dV{@v^R;ue@0 zD}=MG*bf}@ zESj{Sp#JIctOus}mT`jJ_uM!|y(yl7CZN`d5D5%gtm*+Qg+&54w4xVMe^_Qw!T>e( z4!@5R);3&i1>cgx!;G9P8++ZL76u%HX@8AOEgDyg*|R$sXKRYukQz3&o|6#FOe^Zc z=M-gL&Lc-v?k!iVdG#EzE2~t>X|MWot^qZh+GMq0mKn?u3Obr&_pxL^4h3fWfTgpApv1z)4>UJW2Be=Eoib~a%QN!gZ>N}2oF;;k zC={rhK*qclr&9;$;?{Y3MLUG$<2xQeOw6nb1Wx6VnbfH>UFTN zkk=*8om9qB@!aDXY(CHY8rZ}41V!Hw<)WtxDQ8QNep!Tj8i zT1yJR=J5KSr?&emh*DLz{jL~-XRMIVO=t$PxBF;&d$OHw3w?}NpB_Ct&|kjv&F*^7 zweJ6|uVnA_o3}4c19>Ia>0_1s=fO_X{%iN3-@m{Azl*2lDSq)tv|QB(<_@4k0zy81 ztdfox=hMJD%1kb#I4dO+m~X`Hb3?Ns-kqk??j?iu(Nsj8X3>V&9C+{FD*_Vy9gR*f za)Q-AeN?i=MLA@lOo$>XzF*Kw(xZ7Sdb$Vg9@CrlhfLDG_x`;Pj<-weqS`B1*u*?c zRzbG3T~lgt%NDr1oc|l{|Fa28a#n~y=BrJCR_yctdYu51til91B^@I>i6@5ehn+V4UzFO^i(^*3>ugC%T@kmKSj*N2K$$=yh(? zfZ8F=zZLrh+6X~(7(>r{|GxK)XWkmGCGh%DvYB<#qH%^+9$xf?;Mc|HWCEM|;3p}W+Ua--K zg>pbTulcD4K)vRuSc2;)E#<_erWY zbU0xH0$7KI@Zu^rKHTo5IS{(Q)8#6xEI0=72=uGMu!aJ(Q3Fpav!Yc=YZmrFiHccB z4tC+sI6Z}nThj46?St~3nRsHR4|M>~h?-07!k6bTkih$yVe3#{sy@`9gbfA@;g*~n zzc73qSeh2h27rKN8w;{A#H8cTU!SV7;A5M!a1(NPxyEfq<6(R`(T1Q3@(ji>5Z*!o zuc2KR(-p(fLT}7Ri(pe`VS%+~z^p^{<0}}=;K}-v?7(RegRKN7h&8NiVXeKrHh>i! z^}iOKMr+vbb(Y=IJ?0%kaO`&LMzoU9O6`bYfSv1EJ6>!5L4z%u&_w`t>P}P5lydo| zb5=7XwbitYqV*}!+XG91dF1-58Y*=b&Y`=+@NYv%KUl}o&R;vEbJn>5P5D9J0P9fs zXxX_o=vcg?n9~VM(m+g*O9Us=SjRwEG44Rsrg`2t4lp`&E^lLZY+0=@w!HDuNR$D^)E%Dk|g%U^p-dxmt=HkE{6^)Ke zB)HC6jV$E3eqfTIQW6xiX2q_HzC0QRl{Zxlb&BF{)OpLp<0D;C1V2(iI@-)~+V5;o@f2#9IF0z)4`A?#%|;D=kB7SAigg@Q@^y3x8%8@{2#JHBIPbty92F||M&O$ zyAAn&e|PuZ|MO0s=7~~i!cEM1lu3Qx>k=!N2$qcWCHRpLCFB%xqPN+A$ly5f?9jdH zRve<&g)4L=U0muAoIr`Pv0GBqoG#%i)+{;Do9PHp2tZ2&sOzdWKYmm~@W{u3`7yRz zi22YwUC$kr5cV0_neI{b-VjGwzPS#7^6XIG^4E!9 zrYR1Z0zr(jDtv5ZXxpHAlH=eAS|QvyWrL}?CuxoK9_nmkx3=zaseB)2G)ZP#b~zX~ zpZWn;MI+8*&54@~*k6by-jl;yB!_De!Ounl>l;4Gz-d`WD!kEHZc;6{(ek;Yp1@bw z5!1djEB-~Eo63LMQ?1|$^M^5nsrDh>Xv7jWrjk2y;i~vQJ3Ed0-`)PhhxhW|ojeUv zT?}dHs;b;$2)?N~=0*HFP^`*^!5m1HYx8%wC~D`@pZ#384mozYQMl4>Y!6Ru9ZlwG z9&6|Sg*f&InL}}vu?ZdO8y$!elCVM?9TP0sRrg(MIBiPLpu))wxrG)j_a9te_Rsd* zI{hyo#z9RfxQhPo_WMoxzjJU;|L^2+H1BO^K<(Y4C>Hl8#ciDw&JMcG^MPGSmSLJ0 zlr&iZU9n7>q))oj`!nNT?72<)Z;W%BXTVDS|KWCn{_phnAKugdyLj&T|Ibg=8~frm zDf=fva!>C6)t}p-|BL9pc~<%VJbc*X|F`$|_wMQcT|A$S{@)Y+7Q&CgjC@P>wLTsq z)6~L6_!Y?%@DqZK5e%jS7e(-+a`N{GnX+Q2E04jW-Uojx_ycnNw-L~r?tjzqI>CTf z+5bG;Z~Fi3Z|`j1@Bh1a^v=wvr3zE)osT^gsW3Jnx|)7f&P03E-1-r2Hq0bzl19;ttAhGJ~@ z`#6y%`URN}kkb+#r#0Hms#rITNH>e)+>lzVFyclEbB4PSe6v{(T=kVLPlb~cdOltl z1ixAyG;9u&AK&e#zie*WF6Kju-Q`r-1m3ZD&1DTQ!8h$#`P0nl{am_p`KO%2wP)|T z6W12XuXV2K6JTcUFU^@-b4u2jhh@PJ7oyF{TmMOj`hs|=(VAOE&wqU$dZ4%e-Jopg zq526}`D`cOV(c`XDCYQ>)c zh=#sMdkI!j<-(Gc6V|#l=HjWKxax#atmM`C;kS9fTu-9JkWj{6SA!L2zp_yJA@xTE zPeH(Sf?<3uDjVX?YKK5q^xeFs|DdB!H=w$<#eY;d*OTpUD$dsg`p>FC)XOty4Kw5z z9@~hjpQKT_t(%uQ{L4h>-GQG&`W?i7gkFXRdn^lUWkcGcHyPv2S34>QnN}Jn+|m|` zj7zfN>{$nzCEIvY8(3$MvV)yo#QEEor+;TfJgRc0&jWa@Ge5nGvp6kN$Y@YOXTIX= zFBHV-6Q0{S|69!gCOJiJVAcKq{=shZ{{LX-{`|j_N3Gc2H<*<_9+9*Z5(2g*=-ml{ zHJ3~_Ot41<;c~)?%UI};pLz{oIGCKr=w+|=t<@_-b?=*A(*eBgDr)0e{K@9M>B4Q@ zU`=ug*})2)Bwy5WL0hd=$>;7(7jEUbefmEy0j{F|+Xn|t`EPIUp8ns-^M@k5_Y}8< z&@D0=Sj0BFdA#QJMliQ-4#A+x23X-`2M2%_1v;zXAC`pOKcDd2kp5R8-PSMwSx*1^ zJNr8ioAw_&yZ7|}PM*3(Ef*DCu7b+9k!4q7Hb-F1m)2AS3v_0xU63}erQ%tk%4uzQ z2~^i?{EMF(w`@XV)K68OWXEmYyu^-dW`UMN#^xZH!=DKiXA%v_aPrbY+j@wbkG)sF z{%|h5Jbgc9SFd`KT8}rLtVd=^v@SB6?W0-BSzNM+QwK+L4LXbXX3T`HI=9zA@jGGR zg(y>P%2!)28yMs4tKhBfEBKBIc1-1jbQ-3AolO{un9!a515u`{kkYc{mTbG_D7g;N zxM3$|N9Y^ZhLHHl!extQ+eso~?;mVgH-U@MShk|3(Wp%$Qk8Hri#VISrJ!!R_Pb6` zO$yvv<2ozuHIP3yh$^X4WfEYwfg3N?nZG!tV>Te2+QcjjXRfBf89h{1gpLcv3@IcG z<~~b0CLfxSfos7&J9TJ|%Oq(*HMW-9k4x7z%ckykmQAmN+19;7i?Krve9|*1`m0wy zX18X3&I|d(fjxW3c)fltp6JnzJjN0y(16U2XGU-@k`_ zxS))p&JFO5dZu7%!u&XsteDcook?Bf1WFv~0?l@&wYlfxmu$-B3gNkciPDU~;8?OY zSBYKOwR~-h$aQ<+YaXGPjIg%oEr~hiI-2qFC2r2)D(`aZcC96vcI(e&pkA?w``gz& z0qdEh-JHB)h7u znZ~Ev82{8wm$!6hxrz1ajkfNsTJcreYo1~@lj<2dFt>k@FgULpiS^o>#O#yxmfUx@ za{#$LwjCaGAmD8H3(71QSu4v{KlAHpBkLtwR?Wc@Rx0BWGx5FaqG4+akOh21t(9eT z+AY}L6djG?ES9sS&6q;%O-P>egpX&ZxkB+Xo+-IAp4$upuiGT!=29bE;@#%!Ht>JrCNqCEFPCv_dP!|tYfC2lKA$Z`usuI>4H z|9(5E4UN}U(~`fg?5X>JeYjDuda^kWjZ3asn#^}OgwQ>4w(*({Lev3|CG;d$$!f% zfLGf8&c%P+ez;cp?Y3uA*V;F&vXnfjW_Ya9F4!-knSVhh8E-9LZw z=Z5q@kA~|D1S|djn)yF=b`Sc0cK-iQ^32~rJE=8A-VS4LwdT@|}a^ViTZG*i6> zm6GmwzN7mva<`kkYK^H)n7s{oo(;9vO7SV z)()s)ec!L09Q9h&fl#k|TgQR5;Mho6;avMKc3V*haLD`Sg3F)ft|E6y+85f4x7mxU zcgK|-q_0T+O3TNTUM($C8|_G-q6+FNxn@94%Ir`aW-}$Us6SU0_(`zB{r;*H63XB^ znvGd~0R8QqJr~%6{r%m2p#3l7th`##mnic*VJXWbO)9CYrJ!TY;cIIF_|w=hIN_LI z(R^;ne}aJgg$31ISrHZP@?p9qi57|2*8i=l|~FX|?|`vwLs; zb8r50Z~pU1=0EpVKLc`a?DNNcZtMJSGaE9Tz$*R!?wtM4&hEi|{Es_%u66%w<4xb& z|9l?%pD%Crb8Ty%d)e<#{d^Ake@*+J?S~JW@n5$e9^Bvm-pTVN=>I+8|G3)hrzItz z%>kJ4h&g}tgrxF@831p(|JTX;Z=RL>-)@8d+j)4?oa0iCl<#vxR{&k+5u@!XL9;|=JV zCLpWm|Mqr+|J&Kyy^sHYH_u|V-@?4F7$xCfmR{eoy`h}tn!pzslMK#BnFyNUp!v%SqftL}fd_nY>Ad;8n__w@fRo&^-X zd4Jj#nZ>LD+G0Yt{t4h58*L40nm-p}xTR>5q`z;5YF>Rp-NKRd&vriHjpc%wiu30s zO->~Z8-|B*{yk0OByM+2YO7w=m`AYwHfxEZx}>RD6<4)3DK-eC4}Z2cmpj&v&4`bA z!6grQG9YiC9m5+*i!qZJLeiA0+nj%*fw{&5@(L~Bst>K}+!WC2*`s59n`?fyw$>>8 z=QM1;f7W_F?fj=G$**7oR>^<;hfV$8!S=zu{r{aj?es!xXp9zci6~gp1$}Aj+0Ub2 zs;uDVnID6w+vb7V+FitN+CWzz&=ayVfSVo3roInoGoQ4;4%tYXI3FHYCWez+v5fhdq zWSsCJI#tHmc#9M)p)#H_0ukw}_cV*VF3H#!NCw%+3pR?cOpejNYzE{_mdpsxpr!(e zO1m6pWzZX6Dzj-kk_tgK) zPchAURYOA>UX(c*#R(JMH-Wg!y>Egcz3{#XWSV>5{I}O7f2T#vOF@pFzYt!K7yMTi zN-u~bMtfMU;J^MEN!qpc|H+HP=dWG_X>`3lmhb=m-ot(~|5tzSKL7vSJY8~Z=b0Sp z3?SZ7CL~P~LF9yyck?BF*#I$j(Cdw3IVp!h$kSe>!}diyZd*W`WW)<-^BHguM296I zhsQ^Od8aKS2rU>X1&c_Wk-yod$#6#A{b(ahkz*c_LmOdARDjhr>Jbk`Pf~FqdPSM} zG>2SPiV*b{g{$gqDvAhyl1pA(B%DTq454z6V|J@ zu&Fze6Dkcz!djXTu4u3^*!H}xS|6-nzm>6wB@>?KOhrHF@BJ0Z{JA*iL{*nGi%7&nNHu~t>Sr)ky*F^Yko{+V zjIl2B?&QVG7l)@W&R!qBdhtu^h+DRaI{UI{BTKGJJbx(!hVN9jr4OB^7v;ZJWpn`f^ zu!K>;g4*~k94j$kORj^1d|XiAhnC_1O0nj|zkBxM;mO-yi0|tT-6?h}t_B&)tp>PD z-g2@R>;&7McVaTX00K=88nqmU3<+(NP`Bdcg#VW&4O#8yUMKwf#n>w1}bD zc_SUeIk}v~iIC)iv0N<@=^!DQ0f-tE?%2uqWg#c5NO{4wT%?(9tS|gIi3QA(YAMcI z08n!QWh5S(G@FrxXJb|bumw%1{d!z*#WkOk z2^BcI<`O^wqac+Wl-A_@q|9WTvgbJR^J><2an_%gOckbTqi18LhUiQoDu=cJCbPxh zP%6}_tnEDCAMqk&YAl2o>bRAh6wKGEtoqu(6YiNi$DRk95}A?scQco;-^q7uq~s%g z1jN~R!G4W1S@2#d;%s~dd$$`_?fG)T@a*!3(4HX*aW>uz6ix(VKq6}^ljo>v0g@Oo zNn?d*MFAAi+#*wTLRYl%fgta2#esJAmyMw3{M_6kydXsh(=1qHnQO8l2J-|?O!%c) z$=5p0y4@P9sWpe>MHb~e&ZOtP<*+>M0JaFrL|NdQ^LjEhO?etA($0aDnW5B94_+wp z-~ynvW`t<~0&+BRzsG{GtDFP3m#Jm+hRjw_tG-61 zU?q@qHB&9Q8S%G45PEtiM46n)yAS5ZxRpHbiFqy%z1DtQhSOmN)D_|9GDOw8j z71v~hh{$2TsWZ~0Q6-Dy z1kYFL~Nj-({emp)T zmo&N12h}YKJigOxuWMc_g2gjGEfX2%iR18}MLvRRUMbY6vwvP(Bq7aU!@>E=7>VN1 zh!u*m8~H|wfVV2Wb+wjC9S5~21)Bq$;kpRwc`ww_h2%aKn%a`fSdn3>rmQh9lB*pL zgNg{1F)5OMw^CBlZ#jbhA2cUKj~{?Kz&4*%!?6f|#t%JU_!J zW7Txe&zEVXb;iucKh~TXs6OMY8MxM;HLJW>F>4Da>!jHLG)|L*Uy__hb?)@{87Xb{)B0L@1o>LM@A>ac8$p70M< zUac~npINfuCPSo>ZP*6R!n`~y1s$_9!9t!z;!JW$llQj7U!HV2TP-W--ER8JrfqlA z$xqk{)H+b9$^cLul$1y8FHbsgl(5d0`N1z~EbUJvGbPsmvn4YrpR%Heqk6JZ1(EyP znG^WdNZ^Lo+l$JZ0?zlGM~jN*JZjE1jBQZ;)>~igh}&2hbUHp$SU6K+qO(M>hqFsw zTqtQuFnKndNhWT6ja&yjSSuQ`S1f(2R*D8|B_M?O>$YoytFGJjKj8|yjI)Se4#;kb zN5nVZyngfc#o(K7$SIRR%qYsEz|S`co1x@_YpGwW4IA|S@c76Ggj-rEZUlc&fsD#5 zMCmJ*GevlBUo9VyLqS9tPP7=M#IKwS5rbCB1wSQ2E+rCLdGK+nroIM4p}DHfE?yDoG^bU*lsJ7 zT?j)>P5MpLmz}~k)BC(Ek^yPiC#`BE_kyQpdbZ=yJhwml!J}Zuis$; zW11D^Fy{Y0Rxebq*3!*>tBU_lvx$h=q@}d^R242bmx5-IqB-K^z#|Bf1Yy*`%H|GnMqdHTP< zbMOCiCr_8?D;cBIuT#wW)LRR%SP)A7aro+`k6Kg7BCwxQ1)j4op#`&-SVm6HsX$3t za#WI9Yx2PJAO|-}oe=7AHXsoXFCbPSywzp}vjNd7wwK05!3!4o3n>^OUGm1Ty}&F% zH$>7gxtwsptO_m`ZUDsT*P=rLWOEMfVWV9z=y*V;c#4o;y5x144p~9?i0~vLlUOJ$ z;hmQ{cHmMC6d46ik|7N*ylTpw>vv?KV_|OX`rBT?rm-6MkJ#qRIF02%eelK!AEIU| zq*5kYB*6Fty6qD?j|xmR6K^@0uq20zekx-KCOX4w04^u2z!O`L%SjwgKuM0>1WoJ( z(A*&mip5ZsW`q_6ohdZD*XNq8@R6i67i8n&k?@6_B@8dW+&V%k$|uN1%HakH>;i!W z!3x-AHSF$!6!Tq&1F8V%FZ{9tUwCn?=Gz*sH`%)02X_iTLiCO70GB6g!`d_OBUkAWo`Qc*HK;10qMb8 zUnV44Eocp&duOk2tnqGdSju_@xWfxP1;)vDhtG}`5TI;DPzPj06QKs&?JDt-uxuI^ zJcB#vik?vc6Nv{Q$!S~&Xk%&<*>}m`4^NH`zkB&&fIG|JP)C<o&x0e?=2m{6;{T}Y!~HN4dKZ`9r|k}!!0EykrbLUS*-<~#4gfx)h5nkQ^P zd|xKQ593_%6rzkJ6I;ah6F&CU`LIQp3~@6V6Cf+&(qb&kO~;6rSwtqRXy?xn6}<*D zUC<<=>b#(b#TeJUp3R(v@n}{VGYHfk;DJZgVzx?$uHGb~QksDJ*TIpwj0a*?;7#Yo zF^@J$!@2^SJ7J@0APaTC7SEvawQ7)_Y0}pQ7`?7w608WCU*NKf`mP4W{(V{p4Av^l zgh36?Mxo6|%UwO{3cJi;KV^=8#}nYZ0h6?DDV*2fa+ zR-}d6d=+p(k9T6`6>4`JEw|Q*R9!fBD6VT~VE6owx+$~u8{Gqiuto}k93#jT8)T6- zWMC^_@S#SYwUzR~W%O!hTfzPwJ$m%$vki8>!)nOKwQns5aMKDJdiFCTr6_5V%&H51 z&`~MO>Y2872R6x|f=F_bFbY?{G0;w`Ko()lOQ9PNCtQ(hv*b#_3T>^zer|eSWv*wNNuYJ70Mq3 zw%If`Nm{yi0Ym!R+o^e>Rqy0r?^W#KIPs*@oPw^9dbQHKJFjBTbHY#CqfduPM)Ccb zaRIJ73&W9u`~2I7-yV2%*Ju4oo#ODKwzo~dJ3=sb%$w{mNscnfiV=mSru&}@vhEo9 z#JnfMf+_$zs(1%{!>!{tmY+Mz>)z<^I!DLTgU%LF|L%2uaSK=n^UiU>My#+ccbp5^ zw*?)I;?NcaUPv(ul;~7D+V5Pg#DUi(aW*cP5NKM>vl=1tx+LR-VS$_#X{=-^FLN7e zVNkp-sSJV^a4BHmZD@uTzO-RdL|Mu+!K7!HmTngr82E$CaW7sq2?`pmELe!|>Y&tV z)(jnSyf9HLAmmp*)OvsOreaER9%<7*5Mv@3Kvp}Ds%51rF4r#doE22^qV=7&6pFLn zbJv)*NAoyHqpqGbJ#tT9ZoGxnZw86X1|cGxFy}ZUWJpCE9+q-q5iw-6V8vU0!LsjT zBS6VS;8|eKX5bxwl>byD1M<_!OJLQAW}ejqiwQ52$lgOM5(--+&c>RMbj>c(tynZy z!m(MBuW)oWc7fY~c~{Aw4vdFV(r|)PW1N8D(jpe2;-S>i15}BCoEW=J$4+Rv6YTB> z+sI`N`IOaQr_lN<$GOFUNA0Txla2Xyu@>r-p-Vr0;3JG zxH@mu(WB^wkI>_xp@>Xo)dKh>Qj$?VM;B-qvRtyLhHp{zF(BKvYrWWOUDK*Z;Pd8s z)ziL4ud-_1?e%!+;0$$WYh82pI2!F;IHgGpoRhowbSe|F6YM@ z!m_cPIM4z2dsiT;)Al7ma5G?0sJ$y_!J&K}+%7rhk=Z_GtyLSw0K&{Ep>;w98wH>7 z3|O>dg9=yx<0paF(#{xCI)h^v*op#~cfnKE;9Zr(t2R;rs<7!vDPjIulY!0$Uncuu_#Y zGGs0R#(It?gjO!5@pyt_9372}UHE^zB)^piHJ){70YG){0dTVh$c3y=m%KJSj+$fa zuXcS8T#Y$Lj4D6b3B@SuQpa`Hh{%FwV#EqzsEpPHI11=v9ub~tEZ|uXEM&8=7v~Um zPu7LowC!BQmMkNr@MQ4=N(1#byyh4PH(1OK>1p{qywGYU2)kF?8|U4an!{B<5i>1* z{>q1VBB>o&Ux0eM#O*xjB9i)H$qXGINP+)*Ujp#GT32|C)s!DPLE z&FoXfHFu1)jP3M0!&2+p4Vqax2(m&I%$R`|4|#A>4{~fo?N?k8+-$@LE&iugNr%S- z+>p8?tk*RiFW~jNI$|oW%@P4D7>%?PK%^1_YWT7%#F_Xiw z4{Iq5nO3z9kq|kH?u|YIgN7j0qN{`>jAbF~jajB@vB>qrpn0tJ+Rk3Dix6M1nb|G= zwC}4e*%W~gKMum)zC3N76S%0rR=&d0#PQ#05ux;kbi-m>e+uN)W6@~fbIOnl3DYSZ zu^N=PEz`n@M2$LxHtglpQ}OcII<*aT$QBD#sC}c#j6>b$qjb; z-|h$GIe2>tO*Dk%BJ8bF%goj!VC?Hi%YZ~NyKquhyi_GvYf%Ptg^I^O1!2r02haca zfBzqSe`g4pkuMd4r9|_fUFz%-Epcj1pk^2*%(esLXzPBfxrx^Spdsa2D1PJ_FVt%x zX_^b7lM(?>3RkR7KLt}T=l}(>s*~fz67aTK{8MKYc08$fVD){rbImG*6V8Rcu1BK( zfB*OYsrZJBggi-NQMp1QsdQLG-Irk(oPK`RW0735Zg3&EV-G<*NCStr`%%h#$3fHc zg6AM`AZLiK(~=ZrmZ^14Hogb9%q?+)+76x;iW~Aw31FgU!%*}T z22$(a1??HQg!Xmwe?4?sR)p`Nhn^RPH1Fp#K z)fFjBK(XTR<7AqXi4qyoE9`?k`+3#rur#dYM?bYNYhaZnCA&>aih;bAKHr5 zXgk&vE#)ay=YEpR6n`u1<$zTUX{mC=E{T#ZB3eW)gF1>?q8%OyFXC~WReSAMK7>`O zugg`#u^`l#ZQm5i}IKjN6+ffD5n}B_adT_q^uS>}R}~{hZgbpZ8Msvs}r#7qVUL5Jz^l&HbvqVBOzt zzmBb4;JOudRRynZFHzm;7m8mhB+2nkv3`%|=m8zCnJzgtHZ>Tpb3hK0OF9$8TtIAo zc#|ncr7YNo*KEoPPj_Q3HM25F>@BelEIM$kI%0=gpp7W74(E)9Uy(=jUzQm|iPp6W zX|#dj9^&2by$s|3=s+IUL7HMau^?6+)$tZ!U1+mOYi;UPuCDG?Gj68^hI7C@D7flS zg6wLqpc;lhs)Lcx>6s4p_jmWyCNHzY)_1RYcEY)|2(mF*OgV3%9S<06&`1$D)x`I4 z!q5`a!FK3mZ#V0aMp0!`c>0giv$sdDUU*(Hq>!Py+pSDcG|K-6h~OvR9X_k}p{Mov z%?Vyq`jDYBi@_J)yal*JAfCIKRQ2L!mNG|dh4YP(^j=vlc!5iM=B*8b4Egg3;5vx=UV zMVyUKZ8Jv@vsS-;amB(CK+(0Zty8UEeanheRG$=KhfogRgXrphz>kn8%G^;v(DOzS zLX5I+K#sDxkBHW0yVy3$N4}aCA6oXh#Elq^8f+TrFhO;z9>{c>^XRZ{4Av#z$5%kK z;c8SQ{F40ew-<&Cn;2lCGFqzwClrl44M9iDm)vL5IF#0&45+g#sAG1t8cOGq>^g(2 ztg!4~^G8(IhEXeYgtRECT@>P-jo_75T%S#KizQ^hTkq^3x@j{`l&9O*w>{SEOh?Um z{<=j$?+$g5Qw}06%3L*l7tbzG@aom%lYX!h>;`+DLI5ow9EtvtiGCoap-EHXM?9r* z7HAuET_~LeQ6>Vc8;Ei=im$w`vD>yyfoWnEaVhj7h9x~9ot;jt2#YeYUElle#)ISl z=6Z-wGMNpFI5LY+(Ezu!TFUjs$i+a(Eaeq4Wc80~1Or0T=wQ#46UItfAvr5zjv_C- z7?lZ;tVrXGny^t|A5pvK+uc?GD<i8FpvPjMr6!nbCRKciC6r8i)9b~G+x-qjh{oc}Z7me*BHY~^Z z|61Uch+)gjSHdmNz!ram!cx!0&MWNn7B-6n)s3d=MX| zo6$S!|NhH|Y?`wF^}nLOX6k?WruzR^EsZN-3Vaxm3E;)-Nv|j~N6p$Ch2~_fWeafR zn#}w2E)#ZyiurIu{+ZDlV)HmP=}Vp>RRV4d7Qkckr}=Ud_ENQ>R-lh>d^yXRrhllL z>U7A)bE7JFZqV_jcm|SO_2z?GBKEcV@!^z*7fgP5Hlf9Hn41sZB|N;qzp)Qwt8h5> zZ#^C($)ZT{0e6y9%VowCC-n(AZ2J@YYsc}4ST zN*0VkwEA~G#@EYhhYAIbB}yZRZ6)2Q!^He(ID@wDF`-?wKuzStu7ZHNYQ_J5`fdZ& zA6r%lf`nkJq1YgjUR0j5I`oR#0_qg5uXZIWhqRbm;Mp{eV(LfXIIX;OVV2QJP8aCa z3Jm(eUa*VBrta?1C%0)_rnzNE9Ypukdxaw0d1!8d@Hwb*pmug>6MjoHd_y$E+|q8+ zj>hsf4Aoi$WhtDuNKL-n&OFLu`K(&7k`pBy#WXR>^_)kRe%1?XtxA`?ti`ofrE8I? zK_Q3d_}fXr#Dpi&fb4omM1NosmG0`tkC6Mkx)W;{^lQ+bvxLqpW3=rN&5YSsef3V1 zV0Ytm*;l77UcgsHVFXp2Zaai}#O*))7!1ocWd&Tef)!FFgPGJZOQQ^Urt&e5svvr8 zUNCq=Y{JcgvOMP*iu}M4qe2*jaZ`*+*HRyi`hv<8wo+cRG?z27At%g46gQVBRfll( zWYo@DEOYzW)c)OO=Mr{BA&M@~nA?ysU$!r_xN8Xx23mCr_&F;;+mwZ@?#WwS@_;r| z0kvp0LYf&vkGg;m<|SxE@Qy-zkrs;TMJLq_6grjHt_0$6QbpI&S{LU&z_u~~(?=$x zz@$Hg{e754aL1;teXAxIM(#>K@|kIz7NEyezETzUt^r<19{T;@l_~3`gt6aOU_Yxq zRH~npX7LCNUn;w8Dr~$6JcC}DLUtSY5CN;(FfpE*7!8;(@tI*I?ImDk&x=gUTvR6L zPJgZ!g)ct5BOO2mD6lLGsjs#Hae2OFRg6kl14b*5bPNsvu(ENEBF-@QoVNNAES(}v zwl>pWI%duNb9>JX;y+bcU|+v^`{Fc^S8{nDE8_p{?$70a-Fx`(KL6L9JpBNYHz^|P zXu-fZZ#Xk{jW$VyNeiJfhia}ykj`uE^W9mJnp#Vsuird>ad!OX7<5D36ZjwE<$fi?iGCet`8rNalgO_HNy!hH1^Lh zPIY)W3-N)AX)z6ne|e)>d{E8e>5G%UA3bZCNNE0(r|~Ax)#m=edo$&UgU>)m6;QJu z=N});O?)-1!IJ#;?Q6kCe^_HXAHR7HNv;~m1EugSY=VSz+WnE9tluEEwKd<4aU)c1 z1t&U}7^t;Be(WsVfVB%#XUlc|4jFj#1oU?v2HU}Q&>wu;-`+OOl~)Ae!c--jnkM~YZ;w(ezQvOW6l3a!4rm&G=#wW zsq^#mDyqG&xO1?;Qk*;KbjUB-6uJ^?JLZ7s-DLCXkRnr73~j7!U+oD1q?(2=ee&b; zN&Mg0ge5sEL?H8Jp+HyUf89UWZRUU8KRCFL|9K}*mz>}VfR%}YAnBmt&6!qtotb!b z9Z1~XpdLC(BSwM@T7y8^Y9_+w*P_?CQ3EDZqhq1ek*Cxsbgya7{*7lHpeLZ zKf@q*r1Q8R^!NPj4hfz)2BpE@$|2JVkKl(`8lP;{&X7qOEV6a5%!ewuv{2iut0^Ya zI30k6%qUKn;!U2UYD0$;HXwj?SO_n!axi#rMxuiVubwVfVPyp~;aA0B73oN^z&DsR z?M7-PABZf+SI!uif>J2SpiB}C?fRImj?@;4(+q`ckht*sIuoGQq5k_T7|h_w`ZHX?Xwia= zG`UKIv-a|uz*QvFhcLypwy@Ln*IWCawN>a$ui8UKr%2KA)wq&lDec6z{xn0iPRsG& zJ0X}6?fAE$EgwMjt(fD^Upu68*17&B!yvzC=XVW|u>!(ua(==v28vIQvoI;6HHNG> zMQ>grNohvMEb@mlXwJ^ANtfAL$)--OqU_BV%4s2eK3WsXikhlIG+jq0EkIaSyrp4N8joS5~dA*3l*NX+(g`{!tZmh(XB+19CzY%Gu8zu9O)(sA=u9opuI+;H?K3kT4vMzFNv7aGqx@EVZ59Hh=B za)4fU`P|XkxrJr&Mb^j~%i)gK!um`pCNF3Wl%}s2bjbEdM^u8m#f~o3_4WgE#dHW) z-#XRQ^<+Vjq!zX#X=8Z=diMNZd=YoWTYg}DG$c@c~3 z#HdgoY9T9@R;&U@sP}F-iPhB_aUE;}bMYQlH`R>0VB4Ax$grTX+UEEkyfej|mv=yp zvb=;WKpRmUO47mgWT~L$iJU4O61>+wGvQo&A2+Yd-PKw)lV8wm%*aN_vyjTxgTkps z=H6`LHJ#p0>^HI@XE}Q_QoLX%rTIHddh<(cVz86F$< z7w#K>eH!INuDb>za~^$p5>-VAU5iRBD}Gmms>0hmk*dDk6R6Kmpm3&)vA^r%&@wX% z1?)NRg*lS))!7#L3L{Yto{+DsR6AF*vCpxN6?bd?0@Y{x63bM6162|8t_2_KK-lzn zxGOM~{NYxvTprwhTt|pzcl8=8_wOCt|9#T^ufA9-c*2&YgIt;ace{E2d$7NMfB$xT7g2%(48{gt-P;1b?n^J?`{%~bZPEW> zoJDaqZaoH8(f{54Zk_(`Zuk55_TP8%{K4q|cespd$G|O;a_3CBwq2j6miY-ALF711 zbC(Ryvo1f*>DIi3RFzA?Ug+3Zs!RjSRYhG%@smd9kam42K zby|xX8M)q{LZAA468*Pvi|87VBGg5g98ss#qA+y!-63o27i|069kLN}nlKSocFxA&cg}31&9-9^RAt1bfui#cU=ScledwtV zb70oi<YpUj*d5f8N7dAU&Ade&y$b;$KJbcw~ZqU zqrVTI0$WOdwzDlt@+}>m&YbOW+|&LhN&DFD={+lxWkVz+p-mBNg0!P?ve*89p5S}E z{Um3ha3R2pE|%kTPn$myiv$XV0#GOv>T;Nld0ojG7SZ2X3fW3KO;NqwYK?q&1&nI( zft1fKE#w83 z66&yDek$QTYAeh8Cgr&+CVee68)mt>1-mA*ZS+irSR1@5CTDz8T?57S*EN|nl1O}) z@GSG{g-8`P1oMYp$}dh;@V~F0V-0S}t|*`%vruHJ`8h7lZatlqaDXn3P8ThQO1v;@ zOe;~hEEJr2ez|BFDJN7{g~%%|E-|wftgY%4b0dV+)Uj@1Cs4|ser|77Z+r`4wK;ow zN`Th&tov9KRh5+UwNkXPZ!%?MxBOPmn|b>{5hm^K4&_Ng(gnSi#miu7gR3>pmzvUL zfJs$~s|A(R3x%p{l_+kFaLPr?pvs}&8UeWlT`<+qcEvMyxw#ep<=0!OyML4g*U;_z zt+*1fwU%57*8SI93D6pgt^{e`s;h{gMf|$b>MUIr7uD}8ZI)JCQ!MTJ#W(Lj>{NHF zHAT`E#H;_du)cL(Z2Gz+F)A8L5_~jRG*z1^SvUDB zu!+LNs>sP%?q2ycT+_R1k=o7`xj-JbZUOE`{AncojI62EIl*c*br>dYjUWa0r3`9uYT=J+XF5Si%Fh)*ks8`UFwVg&c91 zrg*l6Hhst7`IE6&k0{T!D!;2(hdO?-!C5JOmq1dLe71#JC0$#qu=Z=z)wN2xWUJ0R zO>)N;rZ%S}6*I=7y%}LVI&FI?-j_w;s?K{sD@K1>H#ry8`J}0` zNWltSuhR#^D3VS|CEOGVuLeM#@GK>G`l=Gnt$L#AR%!$JIwl5MM>O)vYP@t`BK|$r zh3k$kw^rZ0obtG`-&WMr)oadrW%UxOB_YMu zD?}#h%L+>F#*4A$j<3prmg>~017^i8-VdYNlZPC*mSEkT3Bge|`DbfM#bblAP;04V z6ifG8vVKpb22G*5Sh2M~S1+``EVElh^JD9M765zF3O%q^sM5*u+PhhBtJ{2~*yzgg z6>b<8%?uaCwQtTXC+F1~Xqw**qIktliH?wE;Z=3yUf}3$U{#WpJIyY6=y5#HSMYvs zz+N0wmav*~Qd?V`(|Yd~hIH1Yf~P01-6othVlwiYA|?&5L#K7k*i0(ZqD+z9^g&y4 z5{{Z;<;0>7*Ir5>cT1UfIj>hY%(H{3no^~FqWA8DN^v--oNP-fagzeg;Y{it%PCv0 zn4VkTF1ot%w?Xkcx5Y&)Xf2)UGP<*Q*cJ73sk$A()|EQR2Gk=HQ`Tujmtni>>Oq&G z0ptsNO;eW!QxejAIzVrACarZb{CY!m51i{Yvdf@pa&EeKE~nV(%OEJLx!4tX zKJS6!Dz*qqt7{gKab-U-*;=WcyGIRK@bo<3vzGstQE;x~|JCcgIM^%uf9>qNc=G>x zjHkqTinJXiH(uiwW;7Z9nr>k{A98h;>L8P2$#ze2uhATWY359m1#+>YrCiY-wxVuo z(ha(%uMIqUz4Z?X2u;4kCN8jO{r6tHD6Rh&ySuwj>;EyHySws)cCuTF>C)+RAh32K z&SB4x472^T-T!MmYpwrSs%jEY1vJY5i{gLm^!nxX-{0vyt^db(O8dVWQK{UqPad`` zpbF7=TnXjz&aVG2At5|l!f{;h!1&?)##Nd>S-Vpf9F~v;coon zeqlp^0mLvPQ=fjOwE!&*tJuQ|ax}#co(s|HgSuc{5Ito{`e`V3Sm$C}tD)e5LqW$< z>VY=Gif+KnwVJIMft{jy}s2@5Y|AZTlrmtL;i zwtz2lV;KdOEmJM2eOY#*m3-7JAGek6N*n0o+{>7on+JeA#GD3)xeeaDdg$OUwe1s} zk`$uZNP7YckMgI-z$c3;cl+kmW3=zY6|@W)?AMQh2NNw?wl4DbW3&v{T0fq-lS*(|jF&AZS#j`q0DxcQEM_CAGS+e8EXEMCTGNL-vEdY*al#%o-k7F8- z9gq-qY(7M%vIP?|uAh;RrVuN1RYXhm@39NIW}Wi)J?Z|()kmy%{^v<&i5vin^uN8{ zZsq=G|LOeyD9>FH|Jm66S+Sp;cL`69l_p@*%>aIQdZG^c-2VY{aqx?1Fzep*ha|)O z?kNOF04rMMBDYl>_U!=4OGl2bss+SkTq{=qdWhq^N*3UT<=-G5GZF)NHa>W^g*|$| zk+4u(DFf9gk%d$Qe(}8D$T@LpT1UYp4}#6%p1<}Ke0sj(^N{?%DF?v9`@ep_9RL4C z@8F65KgRP_`TxV_33#Aw;6ZkQ^-~3`+n*wczt|!g!uYSJQeXdK_GuUT+RwWDf4=3X zd=~KkoxO_v$9}KB_r(7nq2FlV7xN-Xnb>_uXx>QQ2=g;!xA84b!^6{) zqLP>kJR`Aka5;)Fotm%OB}Ke&{0CL|AQ@x#sOXdN8?*|X7WOk8G^OzX>2SeQ{BeK| z_V;)9#W5?75BX1=sN1(TIwdJ2OwA^waH>Zc6D8%qrHo|-X*t3i-I6Hs0f0Ca4$#J# z0DBnEHsDJ04UO}Ux`B%cdEmWcj#xW{onEhn&M_rV4?o+pkn{sd&>XwIL|VkXHBjk zM|vNq-<8+(En9bs)(DjYA`Q@|e+Q;fdylF$Y=>Ev(qW$2-o$ICE7O1)Ldfg32Rf}t z;YiH$Ve$1|3Eu2Zteqq7Yfsy!=YgKJ_`f!kS#Snic>dcxD4+ih4xa9R9_1(6@i9mmo4Vs@aT;9CWEw5ybm@a96STZ>sq0I&OzhfZ;rfi`tv>CGuZNlliQaV=U zwziNJAyL1r#YcafW4n@pk!B-=;~1?1zX>x1a-7u)3WZVdf8!SNtLP{~{m21^j=%w^Kg< zz1Vs2D4y0|giHF_&8gV`J}sC!NtTmDc-i2!F=R4C9&u`lH{n097+ ze|f3mv|V0)Fr~C6O6_(E(j^V85-CX{9DrSf!%-F@ai30;04Y57nw z)F1d+>-={^(u@W;N?3^VjPU?Rq~;j7=>BiNzgxEdKj`m2o&O%=v9ClH*mUY4cH(}u zw$P?#H+njpvhc8&4(j|^G5D+nr%r)?tAq3}aoJc6II2!&mA$u~nkfm`bV}kP?X&%6 zWm#O?l&HHmsBR?jVeacuctODKmARCt$yT_=H3e5~mEQB%|?o8&0Vi zy&({p$a;_C&}*ZZsQky9NlHfaBMIe_{kJV2K_+DuLrsAaN=S+#8WZ31U!Px|XDlUN z8y&Ifl*Q<8N9QP{Dfj#_&ARZfMBnp=|4h5^uYNHZcf~*YC%=ii1)w1guJZ(qXhgX8 z+~>E6_uL=iYwx+AO%w0=e|c^6H=I(Ib9D0hn0vnRZt#2>65N&LQufR5kptXg{hu8l zzJ7b`Ps3%~Sg`&N_Fk0zzjh9Gp6ve~b5}jv#(SwghEJYKXhPOB+ zaBBGRTXDLoN!Zxn&9b`Xjsyw}8noaT{+$$9mxGN^`rS8KP3PI%HBttAl z7`^|ix%)71oa>M+t$0yRIN5}t_dk$mDvR?$w=4EUpHH?lw?tDM<1tBn&uh2QUr2_| zifpGR@rZff{adGlGV(EtXq>aRQ%LKoX@bQ1FnM-e&_IC+&$Q}y}%+3}m>!}H_IcZY9}Kh%<- zk!{)ik{$uM%sBR8v#o#Ac)W#%Bx1Mzr9%USz(O*patkOSdi+yN=l3~}>w7Kx0ShE0 zQ+7i*5{1N+ET*w&I$<%1GmbFlETA~kJ1w}d!O{ZUNCXwY$BIM*bK*N)H+U@UyeYX% z_mgpoLAq$_J`^Bp4LRy`6h@Uvt-vqtlPtHmM{Tt0e-t9rp8wJF&NG~5qIaW|P0@|I z3E%U3es3E{eI*bzT=hx~2<0||v4*oaVdY)|>u3iByiPVDakHKTcw*lc`s!4M|| zSh(4h#Ky=HX)FjVJ0uyV5ebpx0vX>%H7X|1?+QgDMy16Lu>T`||s&H!&khDe64$?<_|Fj@9^~t0oXv#!vt(#7O|iO8$6N zf_mknKhF>+S!TXgycp#~xf1->2$#1s5-ci-*bHGLs>thT%g0-{t!9SHgEWS|cfpK1 zArUKuaJ~(1NSx6CXNu#bTKRnj4--#jWMran;{?ZG>OQT9DIzfthg{&%KCzY zi08osVQ#IG`^VF%3#8L9E}fL%P{%ogELm2hGt_D!d3$NcIMXIUQV=g7H8s(aAY=qI zGEO(AD+38u1P`9-wmfp9*Fmmm>Fn zk7xb5r}_M!Udw#`U0{aG#Cd6Y7U}<9RQ$ht{oN=3??-t`cIW}laKy&iWI&u1X&91t zfLelkwrY#OvbxcXXX8(=P~YGA*6($C=sVQ+cXz~}8?{v#AE1cF`A6%knv&vkK+=qk zs9@geZGAcGMo+@F=5t4eeyFdgjyxTeb6G8hBlzv6DfY2lZ>N0L0IiDV&_I|1l!1I&L?7ZsqID_DsXWVB1-8J;ATd}rtgDL{Wpuw zn#>!X!e1>}PkO@PB#Fe{Ls{&Z;pzE1ey`wOORRrOkhtF*o4@}1`~Cg0{=c_-@Z|sV zC{G)`R#w1D%o{;$8Fw8R%|GP`qrV@%ebX7Sbc(Z#grGMU1zwXN!YM&FIHf}1f(wNq zK?&xZgowr&lQNL89}#B;(>yq5Xt_&BTSKOl6~z((aTgW?y2TugNi5Di0ta#IaLyR$ zH4CmuDv!bQ!n*@&q+ei1^Fb14d)kul|2Ivb!? zs~kV5RM%^xLljXCjO3h5Z%8UA%d->*7Q@np(7Y;u2NM!pi`gPhLR2!yAWu^<&bCfV zimMykC{*BbiEyuta6Hqjm6!VqN0%EKk`Q4oFyn~9NeE4GBJut)7Hx}ZgC$U@r3vk& zAw_f1Ihz1*Hr8YaNi2IbBublV>Ddf+(ti$ zpK@tMksFN48cx9qZu^Ac^sF=`LFT~X6P8Dzz&w~RPGV=L zGZSZk-hY6$ilH5#mXI)Z`Yo?54h+&f1OC=#u3Y*e_j^10JL-elPjxw@CcpVvSx?0W z8bhpRa(+#PC_qQEYMGo9_f&V3l29B2am39EH4=;N9xQ9%v!u}8qHKaeEjWb9sWxdb zGpJ8|M6w{<@=WOgYMH+cEbt+>*ec8ybzuY0V?sj3s-+!s=sA?zd9nM_=4Yhjqa&dx z)VP6dSh5AooR>ZBfT&aF*et9_vJ7pRX�Wb|xFilno? zt_=>4wc0k*S<&F1XPTRA8;f_~rZR7Vf2n*Hbl)tFV!Fa0;+vf0Wj_k-MWLu&J7AJ} z;}j3M8fs;!!}WL!JQ<6Q7#A2b19L8Eov zMa^vFgiw>mh~r1HQ6-9t_PXe`k=||BAawWH^9(1co4;oD-A-sQaof>SjG#)wcvgHd zYm<38*Z0fagO>pDWQ1~ol-fd-_DW^)kUjb7>y|L-K1bz9&K& zeEIRdb2|~p+NE}o{5v#X`hy}HwoD6U}$q|&G* zF>9IEfi=A?pQCFcPNELU#Ut;naI_i&(AF>K=Wmcsh9ZxJa?Lf1-HLlb5-FC>1PcH@ zE;o~y2o{Lb8M=6Ljsl{L9>nQX95?0a)=MzJu^77%%VX&!Cd$92soGvdJ7Q`AkzqB1 z#UK(JqnaIM(FGQ?ufRxA1dcyEVKE3(0xNZE#Z8ma(Nnc~O=iyT03Vf$w6!kaM|Edd z)w`6!7E44DDh|~o&A6{OaXFPis$>w{$mzxpjZ1%-wyGzkV~yHs&`?ZH{Y|#%S&U0fVI!+6 zSh&75EhiTHWobni(dhbHjt);xR8wH5YsB&qsvS8PG55>7ofCvaB|0Mu;WM~r6eoXA z&L4R>ha!k4;O0`2!`w#JI*{TzT(8SW+F;KD_r0pumeO7DW>MC_C~4yCpXy-PXsRT%2p8nGoVj zh?UeiO5Gw}NNZ7$M6q??lm}Zl?BVbR$ARFgFuT?IgL8RmPit#IW9|S0jhx`zY-C?H z`TFRD0hWTp)$y8O5MH$Q-RfHbc!Q%jpAJb1(}cjK6DWA&u~hdG30Z9Q&=^QaViV+4 z?D~Qa<;Eb;(hk>H9Hn$T$!eMty(q5PYPI5OeL(R6>Uka<;uf6Uvq{SG@kE{BOFM!( zf)~@oux)AUq7U%j_P*_T2rb3IRIwT!28&1M9KcAN5+i+9wr?1hWMeDvE!~@WY)hA^ z<67{o_2FN|X5wGzS>pb;@b|u)kSHN3_p@Y`tKbFqzk3He<@?{A{U`sA$9dZ5Oc_>y zE2BC9wB*g>r4j@#Q+4;)f`E9Xm|<>|iP}3$Q!PnpoQ+WH8{Tc*3xE#FXaDWB(a$sv z*)2!s(lt$Vru*E}Z;gggaBCcfh0+x6L&g3BMkrD|+4e*ca05`UE!HXcuh`BbCQjpl zR(08TE6}tF3}2Vf8epz4YjWHcf{HbAX+ z>|7xrUTdwy0JV;P&2iLnsnvjn)@(atEN>f+*fbIHdF#8r-{0%>TF4C*@YmwD&i@O| z^tGOVGb9-<#TF}UeoUw&*K={_p`?F@c*+c^zFGbr^l8)hu@HV9xu@NueE5nU>E2I&i4x(uR zuF%;guCWNQVj^^k5O4)ksN8R=VzS_(QqIIMmClrZ$~#@lE?qZ6%9u-gVCnXt!=#CV z$_h;S_1mqW&!6q-oJ)%C6v1-8U_^?eAk^A6cvR z`di+?Nm9;bQ$GqSe@%Tshavdo_`e?b$bOv7UA;>Twmbzr3biCks}RdsHfzzU6v4HA z`*7_)8i@A|?*O}SIPznVuY{dV$9ftz_h}tkE`^-20y{fK&aGwjvX*5wQFjre^^ryN zYB_3WYBdS-$y zD#{o+eQCPcXw3=e=ptBcw}s&*xP@9O$YN`$T6S#57TN^FT2_R|*49#Btl*C=)GC5l zE{B8_7P7TGF{+e(g>aIH(W&`XZMJzd#q@6&&xNC9=p;mS8Y}IbQi#k&P)%{r#a$W!IRqt-hb#_nXDA{xOnNr!-S`F10Cp~Cyqy^Bj z{gI_x&2q`wsJU*#CA8d`=oFnd8_NY-*(_OcSX{L1q;5+zXPGgXsViA>BOP1UWVQ~5 zR+IiUnW2_8j(RAd8s1_ak z9DL~)JBBXs|K5L5@&DZ4-Ffo=e3S>~qtQ*wxjXUaXnT>=kIr%4D8+tWJ2=JCD!(Gp zn-f#ENv~K5_NwAs%)RnVcw%?djfk<4Kv>z053Z$$cpqw0*F*(chC(vx1sK!5-^e*} zYQoir^!Hgo>sgis>}-Ga8lHQoOQ7d5i?xBvPiJo+_D5Hlfwip8y>E?4*80$Nc_62U zYXhkDXl}yW#Wm`faHrE*V>w|rDX_ftSrU5lhY%&(CNwR;zgRrhx{i_R3bL97Z1?JnRt5`&o;_Y5Gvo--0rkP zLuqtXn+qzJd@>ZPt9T~dEm!@M-uOy6-R_*Ub$OCRpftphl0R&^yv=zAwAqkv!92GJ zuQjFd0QHP3&lG5s@Ze#kk+FD4!O2CpM&q z(}Rtt8jB9v0G;M*nr`S9Fa>co5ihXJg7AZSrQNJ?HaUgR!Unh=lrL$fHc%9tl4by2IjK5+cXxB(2ma3w;YKMxvVDi6sjH6>5P62<5$l%fB5nI;_~El%lQU|DxExyUOn49J^S%L z|9<(?*_$l~e$8xGYA=@!7eLkL4i3Xq!2R*;Vrw4Cq7F!;9kC_6s*0YveLDzA#!Red z+s$`Kv~XMF^ouj_T5Ht-EK#+l*LJ9rGWt0uok)M2*VnmvkQ#cEhlfmd+Cfn@E0o)g@Wb&rs)|&Y;Og zKcoNkztlK@43;`NOr`LiaA8CgAwIECMYKC}CW=jFAl4cH;xE9aKBs*dW1bHx} zWh|Iy<(33s>*dAa*^U1GHI*tg(5Fw;9tM*s3(?-*-rVwysB7{m%u8ek#Lc^U zyvtR}{p8WdMSlV>bet93;Nn^C9&L)d( zC?CYEuuv5|h*@C+xOL15D+2K!L^fVS{#!aD;3E0&#f#mt{a^n{{(GF~tH^)a{I6os z_i!xx0W9?v@NCpzs3*fb8J51X)m(G44`4Lc4BWMuL*4=v*POfF`kUSdHi1(UV4(qA zO}WQ5f0HOIG=8fVTY%^amTR?A=;d(D;c%D*~HF@wdq*csR(~H#q@rWvWx{R8la0K zX(}XR1)b^!o2v3!&bf5An^nH|lm8U!QI!@6f{Wz8{)@8z$3d^Z`y~H8#IPB?Q&%ev7IV)JSV|aJ_;$;DMhBE!g54|#$M^-QY0j&Ht5HDKsvsK(_uy_LT6Ofy88jWH;7U^C9}tD>bR#z;KFQjYd5y> z>Kn*wQ_VZ3T6iVI@)p|6*#G7%Ztl&NMLj!gpXtjh0mG3a8Yr?L_|3zJ5x)sHtwupY zC*WO1j#$D}KcYM%PNv8u?DLlvGi(H^a)!>5Fek{Vq0g2zU6vL;o}RD%tR(*dTf`aZ zsNC+IAPtw$0xppM`n`j){I}oRe~SO~C{J5$QsWG5MkGd?5o6bRf?D&g2V1CR-K(|4 z_R(r>ZJ{2>h;2AMK7%}(PJRvf0fL^&kF6%juj|TuEvdq{fT%;ZlTG9B^hE70+$u>& zNJT4Djc#V_JcGRBD)@C9{a+=#oFZZqwczO(ZFRp?h~&#{^o)F@T!w5`Fdd9G$*-me zbj`t}*ey)D%%ii{GDRXJZ;Z!2I)#|%NUcP4S|p|YQ75H^N<%VYsVV0sSo2}%7WVBc zg{FIY9^ttk|4(?L_ki{JfA64I;s5>K)BVq*JYRzUuX3;7whv&9{M`QmbJTM0{kxSL ze;IxjAdWDNQ0rnsl;mqtfQ5{3Xl#OSWN!Eh7<71gVzj7+(drb{r{||CQs z*-nnKR4NQ}8bLzV1P9l6Ot|l$NE9KlzNKD=#^Y_2k_cz?hJcZ@^&ZEe z*G4fJ%WS%v2?S9hp|p+uw=G}hm1Z&26eyvDKw>`Pd;aV5%kzw-#A~A?NSlKGc65$H znsUz{)2s{sO7uN{_|LQp|LPZ$aaa7KfAX8STL2p3KyW-T$o8K5{5J8P`$K%~J@>Pz z_?xA4>^=W4uZ{kOQ_6CVPF^2#&reeJ3kfpMry;>zSv+OGc>WC!SV+2GeQ_`GEV2I2 zjt^hIJ@%*JecPD7{&x=gyF2ds@4wjFf7<^aJq-R_uXlY9urH=RCM7uu3(uQ|y$$`dywHKd|8S(T+J364WMKeZ@R$-B&Agq`f* za(nkZ??j5F9KA1hY%a2D(q}0l-SMQ82!_|yoUdC3wFT<8-8U)BQ&hm`YkZ9Dw zF=vzbW54i|yCPW-UK!X_OtT42LlTovPqMA~_`uWz*no{Vv!yzqN| z&+k{TxXy<}-bWi;7PPAAFO!|*Ts!Y_K!t>2t0X}N8!WogK229kkhVStKlDv0Dv)zb zKsOX4tgTa!3R$AMn2;1S7%G$Q6kp4XmV|@0`$`9kzFN>-v0_-BSq@jybC#n`>JwjO zbVpa!;4fEOo_AqQT(J-p-${vD(_z8#uwi;5v24}Ud2cN=mHj=Nb?0)$xiz+sk|li4 zYq!x)MKXM5)BmRL@9h7(=bdLb&BSLKS3Yl}_opmG`bdLLm3u*flp9$lWC8C6EDlJL z@vgS_>Lx7g=pP+*@Y<4xrXe9QO39SnkWfw$K@tm(!_yPr^A0(};uw`j**1FqTpg0~ zd0l({9FmX%*+(hi6QkauC4Mg{VOHce$~j4;hKzhnNRY`{laO@8(y8P@9muJqFevjP zV8K_7N7XA(4~iKI>bfg&WToN|$t5|Gl-vl(Fq)ZyS(=k8-}|5c{{Or`q36%vv5X9! zKllEGPMpJ|$~`OOS<*7MT*UeIWDh71Kc*DV2@`fvMqSTPUFyO%B3a;X`GQi0R!V&0XiPH1vpl#qdIP|0 z8zmCwfW;iLkxz)>Hey*y@6lV}ZkiYX*Xgke7$x@GgvDa|0c)5ymop}{&fAhIY5_!% znmaYqOGb#4ERkkn(ob+SDptRizvKatg3x2`4>d}1k2lI^!y-(sGbkbENtuZqhHBSX zhl^niq`ReMgdq`(BQ#2$#*q++O~rH$Nrq|U42`+9Eymmoj=6R=iQ@}b;F#Q^`d&&A zI7ytZ;eA4{hf*>_NT#jR!e^>BpHFx&hCqsSF%f$oaY)T^8qw5YU3*w=qGg5 zYF#N(|ERJK%_a3~HDt??y53gn3gC#vFV^Y&`E8k1LiOu7_8NtM+*X!&)wwn%J0%vkN*D$D`2nTcIGqsGLn1Lt=mu7V=_E z8{tD)a6lL$s3qlpIk(&?DeWfY%mtyLVLX%xm>7{mxvtr$FeFpLg5cuf)Dwo1C=heN zC7_XQL6SYz5kYuDf<=<68N_=Hti}~q#QQsG9ivF``GebSV_BqEhp{fvgD)62$apNt z!Pv*@>IO&0hA<4gIDN@JioK4csSu8GU<4_NLjpIx>b47{>k+zYFs-_R!z;-Uj#xZ~ z10C#aAXse+X)7Q?K&0G~N97!ZigG%sT}aYj4VF;?9I2Tq6(ORnuB|!wgs$#BE1Ff1 zv)8PnNO5!&AT5ite1H?!7TBmbgBBCZT1d(dQT5DTKknSM0~88u3Z{EIAt|v|q8y7P zA4Zf<%DR+}n3Hrw13gQ8AzI01f$8T_M#W}QsD%_2xmb5G3$=)&zFVpR5SRr6iD&?$ zEmhi|(ACS`gO}xgs91=iFUBU;k#TehBkZt|*&&RXr5%;U>|>ulD|b5Z2C#FT%gx~P zXP@5$I^C-e0aUC03CdKwbye`9s{(69N6BMaty1son609R=R`0_G>Y(846U3Za*Y$xAlK!U+806GyfV5D(SSZgIaoMB3Pq{oSc))}AVz{$+jf=VXDSzne9u4zepEc>g^?9Fos9HBQ!Kmf%8F9T$*LX(rbkH) zqjt_h`(nk38r3U+6g#zA%(-0GEZ9x~U_0c*b66)ra9*ApB=xP+^lWUcHvCEJz z=hco_Ss!JB6HiGXm0mMFg|&>Q z_jWi3Oz}*J7M6YiqfwsaaLAX(o%P44jtU?F$YUkeft6qY^wyNH@S)~4FvxoZV=nX5 z`y~E-h>7{HF)L~l5Pv=LtlfibmXHGddbB|@gz%+oRMMzZsdP~_44(w0gxZV5w3QQU zg_$@vRqj&wUXvNpVQ^tC7aH{0YFnD<%)NTO>hfEh%GaG}wx-M?0azGOsXTc&Dj4%5 zDKE^OcQtn#ORNxW$8>)F#&~z5U_yfH1=C!Gp_9kY;s!}c^o6(+;8-vOuzFQ|1@=U0 z?V1)N2W=O+a-vNOpi~H*ETKe5_0dde%mTtWPG>559t5!z=kOREtB`jZESwI9Gh2A| zje-go7eIw#;+)qYg{o6toz(2c7&V$D(N!hDxY107;I0+;G;X5q7eP`58*ReT1U3DN zC>GI18!3r@6SQ;#XbNOYAvI-wjRbRONgqg!U%OT{!pw#h1DFNF!)l$M!V{A<8!~0L)x^VaO$eNW`}*`TX}gZ zE#b74EH!GIQXqdZHY+3_!6}i>44=@|`wu1JRr7|~S|kg7;N6xUYI7{_0IVq`O6PG% z(g?IPw&LiNg(bzkqy-3_Qyk;5WZQNVTTKIAleWz0((~z%z||+{X9Y$U&J)bH5j8Up zY!u}NV^^zay<7-OEUGG1Z3fe8?v73`rn_U(jX^|0rM-(J7c?gzHB)02!ZoPeLYw=r z!6j|dJYKsi6-}hMIaK;2bCrl9I;G0&!V&4LDORw*I?#437YSYbt-32d!qS%LI?6w( zrL4fSBCY^9U^g>63=H6>q1cPZ3360kFVd()Dt{~un!0DTEx%mdX`#<^ZmYEYP;nev z61u#nV7G;-5NuF--5KVibyDBeZYVR?4K2)ods;{cV&k|bGjp9V#fjMGM5|yWD8X$zN_FTb%qZxV-ECONm$s)=xcVU36}@v%D`Zs5=3IyED3nwhdBrY0(68EQ245D$ykJ2+X{bK z-`uz*NkpXrgry-#eKkou@A(By2C$HYz$V06$v3Nl>vux5`gBcZpQJeaCn*mHV7??> zuy1J;1~?621aXj<1^8=baN$SiBa-+z(*$S^(RULm@Bl!EyjhAd2~9$wmI}!T9~@ znPdNdOXHB;@?#YmqI)D{7An;RY2W8+V+P~YGA*6($C=sVQ+cXz~}8?{v#A0Rc+tnX@4 ziUNWo!p-GCn<&+nvjTuf=+=zx$arw|6_t{@Bcpfd<)CmX0^>NLOLjb*B<5$&-`V&3 zURc=P4A7yp(UWXZa7{~nUV%MfqUmLAhE?@TUA^mFli4jxLok%D05`9^j)jERL8io1 zY}7jDUI&pWrqKXNH2&k4>y(k?n)qQ6KU=@^~5L6O*b1-#Y@Uj=Z310q@pi z0c=Kj6jfe&(u#ZlRJ>qlLZSpNMsY^LQD0`lL@YDzf?S>8bFSSD${}F658w{r;7ur0 z#WBKZis2-Arvtv~3ZKPkk)zG)Z+VAjvxsbMBj(mICu*?L4y0%AghW$SK{jdkraqQ! z+le@w{Lk9QDT_ffYOQ(OAXP1WrK1eBO`}KR z8vxjDt2Bb*4}|`9cy@C5=Qqa#$(B5?Sm6RywOf@Iiitv9DB?||!AjzdmP(dd1W#<4 z3xD;gjov5?PQpSJFAg+DA#_Ah$p}(T=(qL)*=HF`S{g57AgvvUHNJe}$8a&{q(2 zq&fvfk4L-U)~FO-RGR?X0Q9?pYgsrRW*JtxL=z_+l*5z<>;}|5GCWA3%;$C_rJEgp zci-<8JEhMUbX3Bv(sjgA*7TCrd>B1G>QO3+Ek$p3=d zLDP!$3uFNe<&JOnK^gBQDH)M8mD+30ro`USd114k5+NhskaUJ3u`JY#mm<&z{j4@b zePpzJ2sZn@Uhm(x(2z_hgm+N>sM^~@%9k=4lSYnmZL9?MBZ^D;=;sNE5x=FG%FUj# zY1zdkV-}!o0514IMyJpuC0uzX8vu@r9GIVJkNOh1z;7U=qY;4{5;;93{@)9vRj7k) zMK9w>I&`h0p&neFL@XYYR0+*fI-X{D2M{gziTxiU(D=v+GjYuUMvP45kpKT-q94Oxithx_p&7#f1 z#|N7m5=&(@SF&p(jo<+3=m~lu@eBvos?@0~D4{IHF&~kXYeu7PvaFl0Qx+DEC2~9P z8-%V8sugP=wl~+tm0Gv8N=QZ)5_BqjpqTm_UP}ij?l%iWYFNIEIJ8&}lFf_dEyrDD zy#-sQETu#_a{t1Ha#Ip3(w%^6xg?42nqrIMep!w;;=6R==<0CDy4|WYszu|L;L@7( zr^*1WWUYhs=tB)2=ZF>h zVN2}Oj?c<*Jsx)*7S=T$cMalx8_DCPoSV6J>cOEp2q@nTCzH6IA=aQOo7b3-P-}X$ zH7vMc8McFYna#+#RD@_z)(tgorhe&IRpb8+yBLWB(v>zz840ao9h^|H)^_%~ZHe(U znd#Nix#@Srk}RKsBz-J`fhD55Psm9HG1CevO^6wFzvdLB7NleE%mrsR;yUFe0AKP% z*~pmA@rXEB$`x>Q@kN)d^7SRkGoU+kmA3sMCMY*j-;(Wq z$g*U|BGc=XCED203OkkNaSZXIH^0X;5<_#I2LT}=3Agk@JC3VDcFo2!FXF~6ORpn_ zL*5lAK-`Upjz};IBGPS(pUDI(Nnk{yjHG80F{=mPL+pw(+~x*Kc_VqcR0~4D4k0iom4=FdV?>SG zAl*36*c6L>KZ<5TzSa6eOK+qOmLqbDxC#Mc3#ecVr93iOO2;%V*4i&@2(wgd!U{0u z2;X2D!8Frx4=yjWX>wWc(#v1i@DhYEbH-R+9w}aR;dUo0$t5KzMfLPjzHU^Swb5H! zZMFpEX*?#p5U1+3Zdwxhk{Ya%>PoTKFQm9CX{^-d^+5V6yLF0ARaCXx=qTqIo6>*EgO#+2JkNwkJf0y6E@Gbhwzw=hV~{7{C;03k zl-j>_7N?It#fFItP~Y?Hw1b~PgYh##IhV1#QWCJ~l*BUhf>@}^G*@4}7Llf~^tsUC zy#%x3(j|rmlz|$cEw*AB6hR|%ZbDD#xRxowVy_M+I1UKcH1W zwlb!MFkH$KH=3bg3L&JVAhuDEdc#*hr?j|V43d0+c6z<3`YrE^2WY?dmZ~1g?bE5y z7ZvQ{yP~+~CK;7A&Xt^3?8~_9J5$K0$`jH<>MtYqv`x?>t#c{CkOwQ-LoHz{k0ce{ zfO6&Jh$S4#kvaKUF&XsDwlO!*mP|Dln~hhmuuluTTrA;~XzqJz(&egKLggJcP*Nbz zPE#hv+>!!K-duSJ+j@6_4u87%;m5O+i@*QlRNZ59C2_kp{MfcLv2EM7ZB3j^Y}>YN z+qNdo#J07bT-Uw#Q?)<9deODIs=NQkasE!i`_8r_k255Z;|g{~v1fyY1HS>O$IH1e zi4m8_Wb@5}-Zcm^J1XNKWyb&#|45Y!uJsew?FVReA>Dji;;uk>eM}3VInrW{l#SDP zXYN>J@q^?eXkBdUHVgt;$#fAnlv{_p4Lb$D8l-%l&lJ2T|!|%gIa2Y;ITJazprY6 zc)cgbWS&$CbF`Jaf)0~_l6ZOgJioW^;O5;cHy>C&2%5>jc&E(-##NYog!MVatdE*C z=gnx^M{Dim%~Djr({7lRr0L3NKi*7N&N6)mba}&yOH2~$A$JNgyL!tdI2s4lh&2OV z#afsi)_0E-=>^HjqQpjWi{1~z2kRFvJmNF&-0UrU#s}0^3*07)+|{SRoDY%ee3rm-N^cy0reLB_?rW zbqMj-Jomj+d30M~etY?E&uI47x&;p06m=t-1G1Ys;|%nsnS6`*_kg=g`#m!NhQB4) zlcXQV%G)@Av@AFlol$W2Okf3*vVuROZ>8;Ls%L!E3G%HtO)S~ZJ`vs1wWE$Ei?Z*D}80EY*eN+g2kmMf8j?> z{E+_OERIb4Um4QdDbZhA=Wh}C!Xl;45mw@k#c=`9urtJNurgwjT`@cJxG1n%^iu0c z$4bRJ*^!umdOsoEeX_2s@*o8*45a0nNIv{K>_oD6me~X%SQrj-rFI_GvVIcO+^kgp ze*BcPo3JtrcE({<@1HT}cB012%c!MFRdh_T0!Rz6u;nY^N`o(g_uKL3-Kk9grqv!j z1*x>;~T%%X1BZxV1%Ik&C%(&1hl1a9pb&B-)3U&j@D=!znb zLRz=s7m9_#w55LlhkqGp9a2OD%?P$@tTOgTc;{qD3g((rP^|zyfJ!3kd{jenm?<}~ z%}zLwZkGp}i=0Bt#r#Z_4DmB07XDIJNR2dJ<1aUI(UjGuOZaeL5Qa7-ext@nU#d8F zclZTZ4NaEwdNjQfCAE-370n)FawpwhujNa~QGXZREMa7&1F~&01!Pyw_Ll|^WbaFN z6*E=UFQys~ejM$83(8y{F$zN*D}!{0n@kc{!83)8OQp*~x|Lc^75Ho}EY6MJD>zkI z*5IDot35!Lwzy0TOe_Lqf&2yCihCm|f(z&=Q);_qwnY|99b!Qm>g<FEkTE1ED$1T4r5LpZ$j@q@p18gHk+w?HYCFIDcM9M(zTrh znPrXsll*kXZeWUG0L2(nM(S1_gtM}pYBf3vqj~)_g{Hy73M26fo4|yvl3rD+wgX>1?DlV=gXrkAOsXiG>cHUi zBWQ$LsZj>4Tp`(J8K@k44&Jh5A6ywJ!Bco>29Zh7>`oHSzP$~(*(-yJ<&BsWj@V;i z8;+E*HKwi@Ot#Xyg<*?+YLTW@Ms`t-s3>wi{0S7E@gvcfouXqx@l~0+Rg^wC|E5!}U1F0r zq*KSsdPu86P_oVO*|iCjWcGAhBiUblipCFZ-iN>yYP#Ifl~5x=l-a$7f51PIk^ox6 zb6)uOMwx=FhQ0$U)nA4yO|eSPe2c;ZonWIG%EyIA{7IFvE%J0)#kDT$36wclm3z%J zGlRXIwnQ{q=&BFMP~Y{|fGZw=`|835Hf>va9U7b%PrlB&ZVI=K+^tC#8ggCpva(-u zD#$#eJ$=Z6o_Nqpn^fiOwH$m|LX#EM}34pVBOs> zSaSrKLZi4{5CVQ#aC>o2F+-+gwHEJs{)~wK+o*)UN;+rkv6Mt{mseHFF##vDk=LC`EA~~h1juSuj zaoMv|l{D;|J9(zp+rdeVmm~&)gNAb-c0FgmmPZt&hDZvBj`CTpI@%k$oq1F6Cnrjv zrcGqh%l*a)=pUw-;m5uYxIjtBb>$x36ZHB=ltpsz%;$5PgT)FwuPm;VBaIFmI#di_ zA91zzdWMmzV{|`SH?{ei$;wE31>5te#)>+hpzjagYZCy`%t>k-SjVq#`x`@xHd8>u zhq1-U#PJH#f&6y!X%AV0l1FTl3urZ z5Wh&CqU=HZF!lz`vyKEynVc1!0g_fD8lIMVi`y$se+>e&FH9u`Y+}+;`LJH^`9CE{Zalm(m;E=X!x?m3KM5nGG zo|`E8?+&_OXS$E0=RB9_Op*3Khk{;%s!v{1Wm>I$4)C(@fy43 zM5Ql}whbbIe_yMRRr@hl!iGJN?$lFtLIEz)maCOK+fy( ze_G`+xf4I_Q4MgR{VXR$I`G_bwQpOzUnB0&bIeGZlGIGHDHRmBtd3bxW7YKG|MZ#^ zRuH<;>lPF(n#h5uDbI#TFKlS}%X$y|e!H?RD&nl#qu!m6;y z%S{`WbpKD;o=o$_0D(-fSx%|=U8^l^4&=Ikho(<#H=; z1hu9{{m1j(Ggf>7?3Q}Dn_MtWuF{n)jeJ8%sT9ipF`tf$DMGfv>9QcqW^PvQw{!{! z+IAP)1cVK%eT=lYkI5Sb1^uq>$xhj#bP>`sIF9>iTdHm=oDQp_kuG%KFo|nLN3`vB zk2SW-1nY!(oz;0E4;j%+8R#R1ZVLg>&@Q^#g;}rJye_L3F7-}Wty+LRHkcV3W*2et zN+PH474AjZ2~pn1Uq8*#7#VDT+WMc<{LOWmO+UO#fg^xVvIUiwZi*aCK$gA{gK-a(S*)G1`MNJUR+U5OdhLyBI{dScNkUWsIii@OG# zOpkzt4X`#&s_iTv2Ah`PT!`h9kkvbe<{Yh5pOZeVhHdSC%`Y=0{pI$uHb+$N9zd3s zoBkD>X8px>S5qkPN~6lG)FoNfD%pqjK)U5bs!Ya;>9$j6i`pi}ZOP4B!)ytOk<0Ne zX?rU3lziLbQGn|k<6Nb=);p(hEpjj+P@@!z`ZDc(DBtNsTfvvpH63i6w05mT|5))= zqx{JN<^nHkwGf4vW`VgQvjZOp0W-e3l3dA{XK<>k8?8-mmbVrn%ZVI+rSP3aq7k1g zYzdl5>qbqo*E#T}^HH$!bw!xf*u=US3lBxg+EBCEi3qdn6|YbnBPI=YFbvenq434t z!QpFTF)I{ycXBg=E3AJ>u?~i0c(edvp~`xo)@)e|abd~(%U^I6-lJ!JD5=hCi5G$L z&;$MxzlvEt?G0Gl0>Ikub%06VuGgK*w_o&0!e?JKjlV!X;ooiijmN$+CYKM5m-rZN%QE*pS2I5-nJ}f|+zPOVn#2ni3PW-!`yz z9j!nytq<=9cnkXC)|!j&y|d>86cZI5c3j|Sqx#M%nzh_dzMdC#GcCEfVVb>jdpNJxbbfC+D;8BD0vbaNQzd;<;cmj23`{F!h)tzh}S%{a1a0Egd0 z?V%arzc3#8WzESSgm-MUVRrIP=?WL~(WuIWxl(DvCHJ}5i{IuJP+}+e!;5|Wr><<` z=XWCY9}c1EOX7{TIr(nPMQ;kR#rc!HTA;V|Z0wQ4@`Xvy_K}H0*n(xea%GIU`Hk2d z#3nsUwy(Kn0;a%7ov}OJm&*&WF|;Jaj+rTJl}secjQd?J8n09hHtfdgC$q;L4r{rV zD>GQ~HhMR+zb-R)-tc{oci&`d8+68Wan|Q{k8qcTbyk_2kRVIlY|mMok^{W0hCYqV z(v~eX^Il+*oj**mvC{nvwkpX6NnaHH1L%C3@H-?LrWkk?6$oYgo$^kKScbs4djHLP zA+V{H!QOcR9F}bb97~^+3?A&9dJo|=!T125uAn8EX@|xwe*y^(2Wjxo8br?!Lh7P* zxs>(;>QO{nq_Jvg&IUtw1PALo^5bhSxZ!oXF2)F)`@9lfX?=g5{I6*=!j7>GxOsVz zIr(WQ=ahZ+#oaeAP|Y$Yjxrl@!ZCE_#ACrb)rXX?< z4quWjdvM2I-xa-s#3<9UbHMqyK}pU$z_lT(%v?w)Mai-6w|S5J3fzQc4U?=WtcB98`v^D z=z>>>8T!jPZ*)P1V=fsy?uKNUV)UvSaAgwjKFmYjW4Q=%g6U8)FK_sxNQFqe! zjgNWqXbrG__wS-@?3-1$BS!26z~E{2rO@~CBWG~_;7vFSn0ghG5t4}SDJNQaE&5=_ zy9y3CJNv4~J0EwtWv)TO)L*%wZgi#z;r2zF$>!J*$T15B>AEjc$#G+Uu#mxlGEV<& zy>v$r>%9CgWV9;A?r9+QK6Fevl_=E4Q@Tz{7??Pr{P6PzG-HxE^N7~BjK3*N4nu3q zGO!VN8SX@Hndm$~(*!MajYpRK(Ya@Ui-;?sj~K`#P+aWfrix(7fRR{KuDgdk zl&7B=zrGsx0=!oLKgdsKDrwg?L;iBhY)VrhdN9+y`)@Lrdr_zqrQly3N-bWhKE8fG zODbM(KVR>+|7{*1w&YyRDy^;O%nxt_M!IQlaZQMh_QGp<`TvfM&s{-Gh5X1#h@8r)gby4+F<57nS3}g?ps`wlK=diHkht1- zMEutZAa+vY=&|;LfzW4$k!Zun?$r^>F%sdF5k%_1^x78Sk63BOI|dd$a4Q(n|5-g6 zE!}7fw6!A3DE&V@CnGRZ)mt$K(tGf?Sv0F}A6Fj&z`z%fM+oVzj|=+@*zWQDp%`yQ zmcD;1#-;;Svoj(}b$JzLVyXzxK>dOk5)2F0*@@7Tn0qXk{32-R+CGgT-&6@O>mr_c zvgU<3FuC$QZYU>`kNb9{JuEqg#BwxeEU7P0W0GW)X|3tD6k%Bh2x@2S5)V=gTWJSQ zkwcOrYHz3?*vn65&W zPPvG$SdB=;98><%Mq^eQ7^mC>d^W0Ts3twP7A+m?2+I$=BsNz}Bo@vrL@B39N~sh0 zX@RnrU8Sm^s#q;eQ42-JUks)cYaD7!+|SYi-ya)~V%8_bvTJpGl5Af50D6FJ-9wHI z4Q4&B6VM&B!|pl8>{yRF{lsR2&6#)fHKIpi$YmC5(tb%n1--yEkspwU;v()lQ}O47 zb=Ap#B|7^d$GaH&923gpes?a{qR27b%rw)q64JAc`1cT=&z!OgOxSiBgFb<6k*~c) z-`f47Qu?G%iubTkC}%E8aYi9thHc=_Rp7}{VmkloS7H?r5jJLd9Kv3deN7$RvcV0Q zMKMr=9!U0Z$7(QoG4pe|EyTR+!Fn=)fc!Uw?D0nY>E`0#kEnkg*41PG!Rh_xWWOv| zJEoVdxT6L$BjbAD#D%ZG;y1lv|L(=m<9^us5NrCpA)d3AOo5y`&=MVEVupLJi8gU zIPj{Ri%KaTZ9cR=$UYwq(Uq9Pr}P(LQg{iePxwMWMAzeO`QlVe9QEfOs3|_*O31iV z(IM>vB{hRpTdMx6(Zx%7uhxfEDxlW=jZ4a2awCPyDoy@H%>(1i&!GD><*-j5us892 zqLiCRXY9{{*U&IqEUV=XG7&(>5MZ5f;VsEcPVqgOr(e+9_fYdQdOq)7esgmjF2G@3 z#mp~=`~zREYOs-wocI@G<13*cu+CL7Sgb!w{$2IH9{1{ek31LLN@x}HWFqCtBc=M# zWdA)vPWJ$`{rK72woe#DoLyb(aBD2ud*vS0`g6DCaD4g9(V!VRy?HDG>D>LLRfYyZ`5g#O%`+9?k`tEJ z5^)Jxz5>G1!Vzdcxlz*jBLEhLI~kImn0ksR3u)z+q>o}JLPv9%=7iZ^4{70t>=+XT z%Uj{y(?kB!VT8fpnLj)D=QM9KWEHVz7+5kzn=1m35LBf|tsQkuh3 zi00V@@yd;GZsI6Vq1&e~wRG!}?vDo_XeHG3EE~*MZqLUDLpcXvOUOpm0yHhz&3cM?j{5*p)2 za}43#Kf>N3%cBJ3pA(3N9qMdW5;pe!#(nK|rA_#ukg)%O*8SH)iahp5rX+g$UkeEu z7~45lA?5w=|M8F}ul~nFBC7?KQ9&_9pyfC6K!ceMf<7R1bK_=lggE_zw5-C~0JesP zHYwY^-{4*Vb3w-5q@m}#8pbx0@KOAzpbuL<mQxDUsg496(P#(D7v{vn;AXJ#8J%p~OokSM!%+_})9R1ytQg1h3nEv{3B7CA z+-NlNC!#F%z>;1TV<9UJ)wJcd&bjzw5Py$om=~0@VmL6`j{?H3-Ew0Xl@1bzhAZd@ zyf^~=RA#j4g7N6iG9%X-gHGz8g55mar|MSSX^$bnq-V4qTx^|~W@po4R#WJ9YWvxz;UjAg5Pi|-FKl7jf6g05YG;%4w&!v7JHK5YJfA*lm2+kNd{M)zQZ7R z@YmO-Tme|=`K=2bMms7PgGMu#|02_kb&}v@34(`hzX9uBo5!t=rbdqsM|J%P%oJ|1 zQtPu9y^9$wHQKtCwBl#v`Q~%?P<_(U3x-3i&e4!_T zl-iMT-i7cl?TDL>J#ns`DyT*1Lw{LR?xZV8CkCOk46HvBs2dJ4<8zhkgA>ZtmCPit zo&zqnd>B;J6%Yd#Ufm?zv1c`t9h8yGT+NoUEVJ2;w(;U2d74~vE`v)%TxP8MjV$%i z=pqcF$OB(iLk4=_!<;^On9dDkuoxg!)aZ+$jxpSqP%|c2_uzVwDgJKIib081YLO!1 z%t!dMaQcuoCHfqbCh7?Tr->Lvh0+qtdmVP(OAexh1A=g?aR?x&<2Lk{I^L*$>Xt9f;kI7OjUJT!X_TLFSeMnl7*}BQIK-fmW|lJ;l>CM*cKLbuI~zHutzoa z6q|GvY14U~Xrw@=oCM@lsHE{2GWX9<*w7iQ5y*=bNF0vtRdiU3s?*!7Rf|x2k@pPy z!UPP3_zkQ^@kSg$Bb3OHMY{NMtKjS|zXpEMfb1jAHKCX%k+-;y~k5X`5)8_y5H z$d$_Ms*pozc;Y|#bOla{L{J>4o~ADYoHE0P|3Z$}1|4Z+ij^mclB+&#D0aeb*qXf~ zBVF0@f*_MjB-o-fsevAcv>q+5L9>n7$!bbjT=N8X#!uSI@4(|AK``PNr80ms?bWGs z^o57~9U&Hopa7>!{Vv|0g5IK)$cudST*7jm^-N zrX#_@z&?2*rChJ?F(gB}JwUBmvAan}f2s}$iCI6D!hnt>B^zu^4B)us%_o&@q*lf) zDz9O?57Yy>O`^TuHUHy9i$0X7lch=~)uf!lIYE3m9=DF(Wg5ahQ4y@0MGKC)D_C7H zn{bYC2Vxf(fLapT9}!3g<;B`rY(-g4UH!WQAV|+JZDwa{IZ0=jpMaXola-nFWBle& z_9r^Dqt#)9AB$P-#`LcE-#AYd!(L|#4rI+LZciaKa-J73?Zpohjhi^0!*Z)0H3f>7 z=Pzy;4g5vlH)~X7tqZq!SX{_H74w-nUXGml0v;JN-F*-Y*CFXjdmyedccvQRkO!j* zT$>V2vID9Cf3^6Sd*&1NRASVc78;uO6vgXHF{BAePN&pb?>4%W8fDMdspO~{Po+L6 z)V{U5{q`#k7_t96a|Y;0_`We(NX^0!B)>su4A@-z5%O`_?f?k*d!qoZJadW}6&`zc zG~oWgJG{Q}(fwh<#IM`)QKq)C4D~)yTgJ;QWX~s_M$+?ju zBh{s-8_Pm4_|;f4*Gp7^!k4VK9%KD5OHE?*uMY!x2!TNo-{1aVW2|LKoDoy$cYF3v zc6zsK?W${@AhW?K&B*$tt#1np@}d(dAzMw~8r>z$U7`=XTHiB{ewyx=xxKCA2Pa_r zDHRIL7rE4iQD+p&` z<%Lby{yGP?2(_(HOv1!(=DK=05-lEm6(Z{4Kk%9stcV78%8rb^DqT~vO$aNKKtNX% z(w489G`W(Zc7PEpViWrPQItoIwP;~QouxpMK~;5dp>#D?k|)#4fTW%&98OY85qvbH z{SkIG+Cz3d)-t9z7^(+@NhqW!e@pBz$fc>)u3#W0=47&InOfW$`sU2#btgiQx)`kq z(zlSi7Mq{-ag68v{W%YT_ZX-zc5Gi1glSpqzFC~vFA>cv_DI$=dJIav9-ClHTSbCy zic4K1&hv25htnn9JNK2%ewN#kOIvOr4YS~1?V-dcL-rF$$X}47E0o zwM|yS_fGivZ9n2-2R>py1nFrby>W-u`e*SZUa;I=&bUq1VY%$UjqwO16%`DAJ{!eI z{&hp2Gj(V<{(wt~L_BI;oiG+9j%S;C4IX^DB9^G`Yu18!; zlsxU~A_xb?xL5GnKu=eQsRS+$WNYFxjkwl@W_6-frRa^vFoFNRzxjQG$OmZy)qYn7 z*Ro|^(kDb6HFFKXaiiP{(`CBA6Ipxp6Y$;m5tlx1&vF1KHN5o%Ex(ONwx#NYH)RwB zItM+95_myUV}8={+<_j2oFFuhr6t<4ago0YM`bE(f29aJA0uo2JcIwutLeR;_W13# zOu-rl&^A?@9Xs6q!UHkd^0U|R41Y;rb`2qBI#4`KL`UU^FxhZ_p_#4Dba)E*=Et?< z%j)*exSFW>w=S#)!l%%rirxO{^sfEr%$R6y2wLQP1Hqz>u;Z>ruyqPj~ktngLIa}VIRqLlz~Tt`*JSrok|JZ}F~y+av=SdudY3*aUhA(kIAHR!shJso56 zxPQGH8Y3W5iM5!951_${@s&?HvPuo0fz9+@d5HB8b(zioIQ{2h@<4SmNbw?6_C`#9 zU4>Z>X2pfc(7%)0`L?C-Aw;I_btToy;dXwT9OR3v=IoSOU~9j>?{p5KfEhkNO6 z!0`|(rTJr&JmQZ1rcze9Y*Zz%xp7+0VSp8p4)bebtdkRk3{8NA-xXe_YrZ?wHM(AK(EopSpvqA_b`Dx|IT`QdK^AN9OzC*;OO zVjYd6ck>7W<-gx{3~vb5LWBHvZ?>gXT`aCVxM45ZS8gYt`J}c1T-2%>4qnPu3mv{b zV#U<7SoQY(xJK#YYk%?+=I{9aac#8m0Tn^-qfR_|ag&$#`}X^O`Ru<}*gMSCsM-xC>H?{>&3VT&nUfJ-B`>~j$rXnHBXWKE<1h>Gb zEQ#64>4jldzpK3J0;oGzLc1r4v0_B!{Lg3iIb~#?8(7D{m)YzaRl}cFjj9C#>BI0Y zZ*Dlo&O%K+PcbBS1R`PvH(BaVYs*h19F6kZk<=XI@v9u8u1asbrW;GU>|cCBYqgrx zAwOtN-b*2sZ`;OlY(>+LWUxehi|~T+mkjx67AX5-%Wk3bvcM==eZAS$L10g?v9p=9 z1@DDPHej>M-%uX8n6D!qPe#o+EQOGRSjk`b@%wmvKK_Mi)GZ~dt!1WXPF=TJD?fJ~TNVvz* z;=>9P1GNjWqwY!OI61nzk?U(Q?@KsEwXM;k(xWo_fXz%++eg^9IY9EmEvR`EH4BKP zWzG&z@@&Oc>KW<}x7lclTEdX9fnn zY-_gZJk%}L@pE~bd~hdGw8U`)eA-&QsIF|@C=a9cp!@GRxHsl0=5MZ=?wB_O_8T~N ztC#!fgf;k2ZekKOEhM{Tp-Cf+FF8q9meg|>lzN;%S^? zV*c+W8Vr>-`{W0n9f~%I)!%>d3AYbV&m>LcNpY(RV-CfYMNO1jgU*ncHb^?jEYm(0 zvd~Ap=pcjKKsh2|QgFL8csX6DH`tG=e0UQz8rEh&VxnqX>1-!EblM`nkTo;{j~|cc zm)knNFPMIvYpaNV6ScO_zgXX1zY*TrdH|wC8`7Ta<397^wBG~}pAoOG{p=6cufNxL zhZupdB_FD~>Ejq6P~z}xJ)$FYTz8k>VJW+xy?i9LSOx$?7i{7 z>+Hq*T|al1$`G@j{leTocdvMNg8?eXesod2StRQmh81n0=1B-5d~grNQtzYc&f7CP z{tLPs4x8q&e{58BX|6rDIeIv3K`1c$wLv*0IyTw;M9@}o`dXrDgj_(U*eEdHz1Vb{ z><{XkFQSJUN3CaQn>%K-Ye{_c=8R`LO4-{OELSg#^y=kbU-gjfW-Kj25-$$VCu3Yg zls3EME~7QGQbTg0Z%-&oXJIph{A70siYkysVYrpx-PDfcC)c|+x2bpO-UOt)Ol#Pp z_qc!7C8Yqqe(G9XpEqsyx^sXb1d%m2P^*v%=fbnU zpD?CnB*vx`Ulq>oi3)kELo9*3+^_R~W~hc#W3Gv4zZ@4fxsftR-OPdeW_mM4oL9|t zTR;{o0xz-Jaous1DvP|hCC*;Cu;C<0^fNSh>o~E8&&cGANeIgEHuV~m-hytEmw;ofjqNNwlgH_L zpH7B3aJZ+BG0sG&fF|Ts%R<6t_)WG(H{Dol^z9xsuJ2=eqyIv&mWuXF9cZASsXdc|Z9W=`$`w1rRF{cNAR1K&yHfS1?3Tl(*%8eS0( z$rbjM(gu%&kZ7?$ZbaL`j{O^ZK7h}&AEU(tac+s$K&!RM z_P5_>JvP7@@NX&1{R6*SN1KJ z3<~ix$GPjiOREG->O6C=sUfLdcbU~8yws$5x|};j*e_{X;muRPUwFnkmsH`O6NCwD z(qoeV%x=yP2X%AshtNZ;#nSjb!n_1Z`gY9eeJ3yg@w-0bn>$mW-%8Y*E$O$Cz?&I& zC9s?&B_#otsTU%MZ;BUUAau_y0oal7XAb(^wAXE4578SB-G24{;Yw867VJIo2PUix zLpX`Jgl6~n_Yrs+?=&&{`*Bv+*pdYxub0>JZfVL8*=~LG{LAn4aPMD14*)TjpTC<2 z@bkIZ`X7ZUVG_D>mSd7BtIzXG{z%BT50oC}V8Ie2?ZFykKTDdh8s^IQXSPvVzH&=m z3cjM;Ro#r0tia0jxD3Ng+as+hS&@c^li()cq(`_mImvw{wnkK){=L!AmNo06@p!VvC@vboAR?AC`crW8W>dW8 z?GtLv$G6PuNc<$Vl79{%xlP%Z!@#hn;_gm62;WY<7asO8AvElT#XR6iUfh*Z1S_KU z)U7qT+&9rNjV!iMGWCm<7LrS4dsHpxDcbOv7 zBQudg5J*tefz0vj2g5K`N>B3L?**VGR;uZF-2x@X5B^L@jh<-tpmUt^#Rl&rcopwg z3&LZ6q&L+Mt0grVXQIOv3-KSSCStK#A;<4M$SXbJEe9Eg1a@BQhF3Qr5py9(=-0~( zV3W(})zebD5&QuF03QGkKXOEZ?g?pj-`oBV77sJ%IOdlh2RRJ~E;MaG&P|M~Alt$S zjs|ru(uE3aVPkcSiN#U6^idhEjMD^MND^xRf%x^V@*6@Q^@2*vJcXdgOdVaj+^t5L z3?1}mTuI15e8YhrOd9+=`?6jpH4+&ep4L)(fmADz(TEitF6jEOR#U<;!m1m!6zr7{ zoRKTKU6EYOZ9o}ndT3e_AI3mG#2IhPN=*<$&p$wde4gA=INpx4+MRT;LO1rrD~A3O zSv_>%_b+kjxD?Yo&Wwh{+eR3M<{co$O%kqsJaeir+e7ElikQEtr+l8^%X07#{Vkrf zpyCk^>~qQupZ=`X+EZqs+9!N?=pDd`qP6&A_k&Ogk|+ntwC`MUrBBxypO`S=OpJW* z---pn7TxOnHYf$oIYQ74@|mKSl!;}duH{&2g5q6~66uUWoWwPUXW7sJe}!pJ!c$n~ zt1A}TTA#8nCKlFaxgEW)V6q!&_OHnr`#o}LuRtuwvXOvD7h$1?+^LoHXV^L<8tLydC}b*}5+uYO>sHd^95vJbHA zY=0wNCmPCha-N!X!Nc{1=VTqcx_l`OG=>8*)WGWL4q2%$3T&CnxdQ=gy04C7Q!w!` z#5`ZWSUu|_UJc1J#QcZ^NP`|Bfux6lJza#SIp&M`$pT-Zvg%#MF1wal;Ml*P#&tXX zyv&)T>!!{5fx6N9rbghgVju^|GY5vq@OGQRX`hYCx=gPQY(0s3Y)9xYULEAt`-YWf zm`=`%k&6(_zhf{*0oEuy204I?susUC_E6{j4t2VXVNY zEKb26+*zxkriofes=AVVO2I)OOpuqr%iVMijSOm{NV!c8_$QSVj#UwA?Srv@UZ87C z4i&vU?A>*AFecYCO{GLeA}y?rFK2s`pXBw9GuaSXTx-cB7@tx4>tHl`OFDC6G+NOGt-?|nF?}wp6V0G{;tx)kbqZKu%bN*Lp2`Qo?iY~Rnr>cd>XlvZ>B(+<_1Pm0xn}bRs$oh#I)c^6f*@0 z6}g*Ft1k?(kVBDXkh_qVVx&RMs^e4XR<8vXs0$_P)2G0-Gt3=zKwTbpth$CnBqNy_ z8G%!wmFSm_4>kv4L2KKcbCGWMMuX3&5EZu*cL`FOG3-?vtoyR4bO;`X!Ts6O88K1A z(h69rsz6~KY2CEBqO8cay)ZKLU|nXsKdf$H5YH~J&;cf$s8}!_aWvq-4c2UVYNxD; zTf}aVWMg;|)KwDJDXRBuXP>~_w0Nc7xeiZwLIy=>2v^MNFh}0;vp%=uSZJJmuU$`k1LyXiwDJUq0)pE=@Xo2rlvu&&bUP@)o7vUrNBkw&!{q9 zO9v2^=@osb#U~GlcR4Hz$@LLMx;8J&CKUHYG&KK*V8SJ0W9k!^=V~qnQ@&cORgI}{ zG#Udtgd155mpf3sy>vCg)KFo>`KQNa3*y>FR3vIGzKOJ1$AnbqjY)*WqtlAPDN&tp z`IC4G9K+G;!Zj8*|LSoC!Z)2v0mUj$@m;}_itrIpAU|Wy@Zavu6p#}3&y_+cV^c(d zbLA+O#kDS@Ut{CRCYigvN2B|foMKzO!X6{|ZTy>El3y4$^haQtcy_VD|bv;`# z9^5-eeQW|D0X^Y5@*-WO`4O1z{F$m+n)TU(39>5rD#-)ZZ?0pd8A>~gmi{9!RrJWj z!&pO0)-JO~?ct!s&0q6xop7>?rThp?baKVLT*>dGJ}Huke~@LGV53S{QZ?nib@OL5 z>7n#K>AHRdrd4qlx+6ksDxKTta2ge=Se%sn`?2}_2`}}f z`>EBO)py>DR_S%bm3>Wx(?1FcdMl;E4uTavWf5_UQhb|!t@Z_t+q%So*A=#qM`xMi zI7o?Gp_3iJ4P&a;!%LYlTQ!hlSWDIGiWPN)n!%e>x?q^G4$#l|GuS#%;%w8(`WR^c z1(=fXdGC$X@i`a-2Fxaj~yZ6s9d}4LS0Wr$;7Dlp7t;7Th6zFjufa5W>U5w5_N^#bCue$+kqV21DM; zEl~~g9gGb8ov|$rioO08Uh;liHCePFFRs@lBG=A4uK|Jc5r~{RXf)9eD^TfAgp`k( z88oQj25W+ReJ%Cb(A(=c^*|~PE~zeB4RuI-*9~cZ;0=eX#b_H?Bi6ZYt_vX5teV5L zeZyY=CqN>A7x;$KX=pGag|h+S+n0rUOqSFZc0rrn)3uJf+{j19akW*Nx;_)F(c zrnh1P|M8cuwafqpa_NYn($pINPO&cCJjH#g^IX4o6$k{ZUIBR5gj+{`y(y5wG2@o* z1Vqi02UPF@s>AD)5hAHh?Bo($@F&v(aL;-ni?IBuZ%rG*Z zKRtG}*bYoU{(VQ+IhBRa1Q4BN3HqwwfA|j4YZV4%O|t}~nb2QEpbct3pmYFqNHMiK$^)aFfVoRUmeIn&bu?QU>)Jzsf=N#t5;GU_dkYM;mQ;V5_MlH7pR*M^+mq z1m$)};t`SMK%a*`MkDc;nJ^=<3e+Wvn!Wk5`6Q)KfFID(@ztT1CU<+?z(1X_`rq`H z-yFou^gqlSdPbvl$eMcVkuEvK%ezuQ#@jeb?wFg-CgRb$_v@1n9!GSh8L8k zQj~Ln4Z`H}m57Y}e{`Kwa3o;&rzdtM*2K2$iLHq`wrx&q+qP}nwryu(_2&EjwN<-Y zwKsjyH(lM;-S2yT=RD8(1keWYmMJ-8X62&tq1F_uI1D$gv4)vRcInfRs2Vrt8sqx6 zCk7Y-M9kEUG7|)-0_ck-K3L9AEH?7L3;QC1l5HeHQKp}wz+|IwGXT0~=BRKOa)ei} zSZ3eA`YJKR!4UEADzF@TiRTWJvjtg4sqNMjoz6^yc^FK-S5<-OnCNs&nfI^!qoWMs zU@yT8kq6>9P-AXfx-V`r`82`CADjiEb21XtRsxp23Q8}@wO2(|ixfL*R|Q}s*$>NK zzJZINQoxAO#7%r`WS9OMDN9#yf)R-&r=fo_VYGjj4wz4^(2|$sb*=e$6EK+B>n$fX zUJ;3=lAdpf=^;bRR~$IgkdG!i1OOjTcIyL|mBecE8NY4D&RYuBh3h+;GLzd!!^rQ2 zZXz+Q_c6>r4~KCpNZJ9Q#JNurVKHlQ!c12zCxsDZ%-;4_{QSA_36;pM#Gz$!sJBD2 zYE1i*N`QQP;f9=Ic-QwUenm;rL@PT`QpSu449vi$L!EOvB(E$93w3A2`FQ3F?}T4^ zfY$#OtUJeu#RrE1KFfKKJpSp?og7c(y9IGm6Q+Ko$=F^@#Y*2z`sAGAFM3!PWBfZN z99xO0kO%9q91RYyT7k3FE+Np!JuGUBS1BoD3H)n@eqgwGnU<~BV6sWXw^G`WgcH^L6+UYOW2&`ciw9$yq56tvX zx<8%VahH8aH*J(v(paI@!@3di-IOED}oAShrC*qDpU1abkUzOXHK@|O4@iA4ucSf>Bg1>RYPh7 zgppY|(~XXfLMFMd&{)Xd$99Q>y&s&t0ZpLsr;0jFcoFpKXE0E{=(qhr`uEjHDeNC) z^c(zAcX1Yhj-wGwh8{KT?L5`<`SBnYoQI#4%Fvqh+O0V`^a67QtnS`^ghuvxt+}~~ zmxD&pdfzK&nUw0R8FEQ1UTA-pD!-W_LWhP#2{I%Ve;r# z4wPrVd{oS#G4yLckVq97<8FurBvok}z4pO1=J(*$QnIqA@O+R$XJ?}efi?;ZZo|$n zY1LO-gkkPfp{wY016*E^pg-B6!}|#~N>bJD=kIeqglD}Zsg}l1kYGy`7h4rjy+Yd2 zKJKnI64oQ_#s3uP{uQ0iUVHIqJ zyKcjTZQTFKIDJxi>yDlh$-@axc(Ag|WJf9%6^EsoIi0GZ&j$B<)ULFKx5+riNnFvZ zkiBkTpOEeD4G|en{(@+{P!=`weSXU3gfzk>BTOG@^9%>0v+x77)dtr@V1`(i!R)FD zu4YxdFQj3f;bhYG?_CLCZ=t>1^bmo|(YW$#9KQJ9@{bt1{Q8=%Ln2wqav95JGFJbE zvP1&KxFhHM_=1%hKAwew$sQh3AQBn^-Ub}Gh<$QCfl`{}O=eLti$#lNIHbh*3SV-* zNoo;qWJ9?0?jA4dFJM}-#AkN^ubi_bdDnd*ZSbC~6V=W`9Yp~opNv&j< zgjqIj&r?o$u2(8KL#7@-SU4o(ri9AND8(wNx{)HhebIjTePy)#xgP2V6`fG>2eQxX zMus0VPVQV;v}1)oYAm?v0KczSg#QZ6>S5N6%d?P@PST8uH!3h-Sb>0_juqq>Z2c*; znUbpF{`u1<{3x2G!9Y5n0zsPy91dTV`ClgIj8CuXU9AEdLhP*wtz0+_({#-TQT)4Lz+8~l} z?OWu)U^;RC^T~2f0}@G)?=Vf*A3U2QH6>z@h-u8ds~cKIf`F*?_;|2Wn!Waqo;8Vd zJ<1OfZ5J0gpIArG+TadZ|(5dy(KW-THB2xyq$(j_L$X=i2$$_%zk; z4G3-X`CN|qL5zKDxN@sOwpL}P!Jh#GC4Ky{sfpTk{X#!A&V#i3=K)+CwH1b-o1o#g zr~X}#@Xf*)P3|p0bY{K3P7TU=Qgno$2J&Ln@{!O$7_5@I>FEIWk{1Z#hx1;)7}Qmh zhCPq?tUGZKE#DtyhOkgUXY(Y5$k^hXYZqgT(+JW@gJRv2O@h|vj-f6GsREFZJ3F>- z!Tl?mcO2|gnvv11s-G|fXBf}I;Yz9bVm4#zbq3z&gF`70Id98J+6Zn1A?TX4gZP+*M`xyuP~2A7fWh@{rL2D;7Cqj-_-B|=LWZbBDZ;F@t! z?`mH2WH@0Tmpu+?)5_I(glAIKWCBUSQ~NX`8mvwCdG>&Q zdb5!ggLM-9`4BA3U^}egY7wq=nLL-n8!<_I{!ez7v(Q~GA}F&c19TjKWfJjWqEGLx zPJRqR4gIqbwX`2Z_MQvjl3Z}1?;Cc=7={^$v^o3=c+cpK4Tl9APTdr5`1vT?_XkV^ zEVH&Avs1|K1x^+0L8kcqhB<}jTvhg{JiIoeM&mG>w#fh;F-(O#l^jWS?s@EISa?O~ z(~+uYajplv1%Dzf0oJmWZ(=28NdtFX|7p}(VI}6Uq!yiCLI4TuN8o)(f#J(sWl(9f z>^s*Y<6;4LI2!oo=?{2}DMJM)%=AZ4quOAln9z7}_ejNWlH0E|ma-wm-27}}wtBz@ z6ut3#$;Wh3vW`@s{kaF6a-og&sss(CE^^%y4qVlO&5R>y`!bEtIVE8Y+a_M*Si(pQ z0snoa*2?+PBA_^pTJxHv?aZtiSpH>>N@J&si#QHHHqIfiruw~zNKoc z>akTRf(JLA12C9~MLLWOOZ6@^|DK`YzE07zMu*Ek$8?`JHPEi{kpPEVEnc7&Q z2zCC&E=0&EPIXA3U=xJn*Md;e^2<_V()etFm077IhL>iko%GPQfmgbIE&djllY1fM zG@WuOSin6P_)=swHL~ngz~+blmp8z6Ue&|e(9hZT zJ8P+2h?^My5iG>$CYssDFG+b~d-%wK6+2|tcZndmO`C7E{dRSqFXcVJxwiQ@XGDtR zh5Roc^!`4DI$bhbm3%#y-;_(#y3!%Cfe+=#xLA$_vv z!gZW0&8OWU+U9Do;XJ)n+dMkx)^mUbfiqEz3yIs7Lyo7gC~@7D??+MhX!$|ua-B+F z8`()(Sl%*N*cB=`vx@5iAskN_WSzZv z;xGnv>tpho1q>tcG^E(~nG4_UnH_E31ODE-pHJsK3tfcJ+c#lV&mVYG|DN}suf69Q zNYI1dzrXP3i0bd+$CafWC?%bVCVul?RNCi6smjcHSDE3}b}nPE#Rgfz*+Fi3EmSSD z;+6QFJv|i|aLlq#`QkPe;%aL242H7Uq%}9=?M*?(`!2!}6_D;*=q7Xf@pD0Kt?RnB zHdWXK4g+x0oI`p`UCnA)m+LIoE0m+6rk%Y>u1=B%;Apz~$%G6`SG|m^7_<1R#tPb7 zt@aG5`k^GYRSmF|osH~>o$YLCDqF2(QKnQ9bZ!Z>3Dpq+4!<2=8%F29o7f-7WvA8m zB_`OnEz$k3J%lz%>Nk?vqIz(6^WZfx-efdPA4R`@fbqVXc7{qj83gY{l!Gs{d^>(j z*Mf%X?6c-^R*1huB$-mE$ZoYU#(Er7?M79#_Ys^^XWjO`NOw#W3%P<}?BL(PzVc)S zm!MuX6FIC}jwmwwzo%0RU!=Cz#(RF-Q}`@yrh3GoJCQRkgHis`uExc`gr}3nnYN$q zBnqaBTEM0isw55ARFZ)d@!EMQq>YteQHWK;JfD@9p){4p_j_(%U93GNQzlZ*PXyFD zSmlv5wB*tEG0*S}RD!#FD1g79rpCdoQwi$jdRmR;ANJM-FjWZ;yRa_L)fG?@^UTxR zVOk;m^790D#@9ZprDHdeQ&xsR$B{Hh_k$2p&^*rDQmA4{A~L=3HLPO%;>W|Li+*sbnF36r|-l6e)))|of zyMvSlTI7#HNmf>M?y;F|{>CFFnTrz>B4i}HEy6N^7|C+&D@I7=ua3UolG=7>2rYI+vCj5#HLGbu|W5ChK8r zm4oZ-2F*dK)&$G-hyxb{V{$sY6$izFPQ;N9#Zh@hCIxY!s_}xg35ejm1TrMbOr>Ul zbI|w91=sG7kC9yBMS) z2HL)g?d4Bm?XH*lN`y>1>!sR7(7t+Q#$w8VnO65JP5|o%0ZX0pGZFb~_ve|2`w;Hx z%?=hm#bd8F>^4Io40SQ9ya`@I^nij2&saW*_Sg|wh^l!!DB(u7DU}Wf?CSh5LRmen z>P*He1T*lrHKcaNX`QGJ{aAE!OF5O)79o-#9?lPW(S|JTv#T;r%tiZAnQF*fK^wwP ze-5LmI^GfQf=F9v>}DcY*#{?^nCh}0Eu2P}U`?uXyF4|F!Aw?K=A!U2#!eQi-_F`K z{^7mO>nHMKM7EEy z3-S(ZjpM{c(nx6Yo5m>CUx143e2^LYdXJL~1kEu29d%BtQ`L9qyCLlMD(THgzb zuzu=I9oRWPS~DDZm158{bh$Q{F7ch|9A%<_alo8LV#d2@rkJ5S%RB1SQV=b``BINnFR(j3>`3b2UkPO>vK~@YC?p%`#>WZFN zK<;4P>46|2;)OaJ-JaF|d-dj^kVfR39+sV(aj>^eR0ujzI#l-OMp%rM9ZRCl_rYR} zo9KtCw~qRHEYgW|z+GgognoI|sppkVHkZo#Fgtx)Bjb@7N+7^&f$P|0p7?T%e5xRm zfJvJ-!9sK<>#o{0cE6Sh)F3982^K_skI_zHH zY>o0O#vyIDfJ30i+Z`j3V1nPeG7RXlI{s}?zrFj?{%Fy3$9?jNH3=HAi+5nT(*~v9 ztp4Dg*O90tl+xJM$S*{l&6`P+8K<@`QDEHzIjmNmuGE0F=92VS`8jC!c7Hk~&p`AG zk6kU}M;BW?|F}p$_N~)=MxRSP9@{69+Na$W`FI6i%$K z*DFdTP4$vQ{l_@CzLZunRlk@?NJ{ZZ4J^-~WcDo9q;=qEJSHGbmRh27c=l2I=@pt6 zWE6KgG&dNtk1f<>o~YtH-$t*V6~XWT#px%&&%S34yq(d3*I+aZoaJ>HvTG5Xcz~c{ z4Tkq-fg)${4Mw%3^bU561b*_K^UjMeU>x^UW+8omp+?M80 zAIQ_*3XNpf%wQIQ^60jlS;3|OfnYkPjT%j1a4mVc*>;>^jL2f&Y=`ju>#H$LZ%6)7rb z%n35SnXBs6YIaWS{1ZV|$QxZT`S3~`!kwg)EM?6WqK+&V0nnOcrXh7f6&$nnsnrUx z!3%LsC&nPsgR$V=^pcc9{F?ZGVHXvTb|&o3kEZ;wT+b#fGfEy;%P7O$XT!!s@&Exm zaHyD%PY{@B1i1ZOTj$Kr-I}^~GZ0Qh(-7|F+9o7b42PPxIWPxohG9Dj<|tHEddDX( zB4`axQ$x@KM7X`95iLK_RRW3LA<_e{U}Pg z7qJ{au-RK*{z$VQ8;KV$yWJhjBtmG0Wd?QlsqL(gAXz&G>)$YK3Jzo=(Ca|lqipaO zXgO*ia#!%5_o*0j!#&`5H+}hX3&{`9yvx`6mEfZVqNJupT546b{321$A^39jn)MC7 zvZ+5*y{2nI%jhW1+EJBpt9fk_w&*-dH}6mjX&+&+VGjc6Mj$T<9HK!8sS! z=Yuf1JD>*#3siGX;?3hlR5|SDH#3~`Q@39E zWv%im^S7#getq5=$9TAMaHeqbOA>P|97Vf!k}__;;lo;fI0G`N4meokybK+(7DK)< za7H}hatq_?x>ZCfYBvqK{m1*Q(V;miXf8N6AKaE#DRp00(TxQf+EwHyVU>Mj4;1u| zDYSFzpB@#xq4*@}E&gNBm3XNJq9i{eU-X6o0DwBgDdO!6KaFKI%RuS59~%0~%K;X{ z%yYHtzvZ}n*PQ-^RH`f3Ke2(0B&eC|_*FYmgb&2S+lik4(-7<6D2JE0l`L7Z`ukD8 zVY$<&U+xYC=U9(w;?L#^+bqU5@6jMVS&;TAUn};3rK!p!!Zq*FdI=;D%;(w zm>JGQUvKNo%eV0SiYc*$H#+M??xOa3Q|>K&xGUjYGc#jb1o4!ku}CYnhHanTcbQvf zA!R#bChFC;+moZi+_uK&m-Xq+>gL1R+s<42!L>QB<~C5D5NmMLs_Jv*@4v5` zv#)~@*Jq!WJBfOUh0bqVA*EyvoWgh%?))Xu{MWU{PrZvKcKKoz6wL zfIwct?v%;jgwh4PSWKjlKg-fG*@+^Du-Q45oKJjt&&n51G_l~v0;q|_p)WFCFVpDAAn<_85JPbm0Z@59))AD%~hSUtE{R`eOnu!2!~ z`0(m3jLSBeU&@{9Y^#ijGox}sQ%>%%K^5e?QR^~GkoBh5!?WUZGShq_kdB6RW~XWd zp`8MJy{%0HzK$h`zq_6Qt$7it5#{dh=jS|3OTIwbHh53+;H#uUl|U$L@m3YG-kl$N zcI)e+KW*JmB9gqui(|nw*X62D<4U$%&Jmkz*bZX>Z7EjZE9gxz6;*G>xLTb;mKSv= z(wA31Ub`~vv<-E0vY61TcRbWSX55>LfDq5l>KV>3c`LKP&k}h8_EX@&$h+$UP!<2A zsT#dth%wprg@K(9Q&y=Pq3*uU*}4c(v}okCNn%1ai?)R_x!iv6dI0fP!c0?E*+unM zHaqrX@tnjT%$B1sj!wT7(*WHrAi*^OKI?1G2vo?=o2T!$c${49J`GW`taU6lZ~SLB zkyj+^X?Jg$%7n*rGNX#R&zp=x9$@x8o1RA-nq@bkDv^$Mhu26sWnj zj;YgSNFyH1ro2cq@->@^%T1Ufy2=fR&4%SDK8H5jX3ZQdmR-)gyZx&C^!c#$^DuvI zRcxV$VZuS3Qce10(_;3i^L9MNR{Ca>E%t9wUZJN%s7C(}FJlLKLr69BO3iu)m|sm% z&(3?zn+|kC<*y#yDDK~EuIfgFbTGC zKOdBQVz0UWi+*&Hp4t(y(NsNSVT&nt^^Q9(Se;)Up1#mDhUC3Y_SG$8`2}{dL$Grs z%k@jA8*b9qd4Nf;<~#o@lK0Z3oj=LpUz6?t%q8xho3+X+Xta9RSUD7YnBLdW%8lvS z)?ze3hS>)v*s#Y0F?z8+J3Bd|Vsm8LbmA-=4UVHbwt}%8JZJ{xpzgE9X|b49)VAf> zEB&V9VO03dv;ln@{;SDn26~k1uUtBv&-v~2?v#+3y$1sXWxKa{*G%Avq*nVZi@iI& z2$5gSt3$~os@cKzcM(*Df2I1c5)}uqWUVI4=3gED1LtA*6cu{t4-GhID|LI7#sRiJ z_FIZ8U)gMH_!@A#s0x-LuaiBq514Pog|c#a{IwOwlk1qWWm^`b;_P?f5G+Il*$NWV z-hZW>n3x-4;rF*7sp8xytiu?Yvyx|F=+>*t`SuzFA*Dm%1wD)zv+zx=F;*Yqg>t2p z5HvhVnG?2LBX+_Vd?GnF*k`3a;QGl@jlzmrqhjKzpiaCM>n!CA!+1vLy{?b1#N~Nk z`e~{z{;KbCaJ`73iRF+=Lx!LuHVMMnbh%h8>&5vMr2ZT3iq9C$qEH$L%VIh#ay1*) z!FZiIks-JRi+lFqUoX|F4CbBE@dO*EC?uPPW-aIVS*K!KrIyr)>pCVHoR*P6Xb|qO zAU4eU8fQLU>FCRWLAFpv$YYeg8u zW~_IVy@q~qiqpwO zoAENwBF$Jp_KTiosf+C4?0)hXTQ#T6+1>M|2%+# zn;Ohf;oXM=kbKcTCwW!VD3RP2f5*0>4$78HfZPRc6(x8D8<|&f_lkekyBpgo{&G}l zh4muD-DV4}AU}{-IZN7IgtaB_eMz8om4u9oJ#V>?pnnumiBXf)a z|B7yHTKH8esx>-a$%o!Pj*`!rGZ{qEk-S6 zvrbJvv>$bhAy!%2X#YGPP`_P@unx9*Xf2YrC&@J8eqQ^diHKY;IZ9wji zCO&tx(E{k6eP%{&Un(PjPewgQ!1u$ugpN<2Uo#~c5kS=u@)0aM39?Hq7RorH%)9`Sx|MpMzPMuB=vCz zDQ2yWR_Ys@t=f59M3`a%MYein%lEyvUVIRm@QpVtn`1V6x}|24lE;>v50~s9G24zv z(Jj1%1nbYyhS6xLR)?>s`K}dTKuHtgI?Z0i%TxOC9WuzKX(4DiiOP`ot%vJzbsd zrSOmci>LG^3AFQKWi{@>Ys3fA+nnS$V5HT?_=KJkcNP4tl%QD<*o!3@ z6t+Y+`#>Diu5A{WZlL>|A$AVv)`8Tb0A3NW9{32jf85FeJ!xWuzrd1VPnxN~o{E5i zRYOF2U=|~gwzICXi&q{TzvpfJsat+d3=3N0fLX8wx*SBMD!3!-4#Q;BnMK-|>l|pT zQameiIQKwZE22fXnBPCEsCjx*yAW?fuNK&0Fz9CCZZOE{bd~C&sdx{Re#I>>6wixiK#`gg|InsZm&tcDzh zRHhChBWufg)T6aX&YiZ9HjEPf4uA&#>7&M?AEuem)^gi1w066 zGRint1wV5Tg>Lfg9IicmoV3{v{^(i$yy@Fp_|M~4!G99~Tdyx$43>`y2sq3M_XXcQ z760K%{x%A!o}iCJ>X7mz;STX?w?7=Fzb%SUttpCtEL_lt_)za(U%f#vg}ll{Q?IF= z9Wf0~z%KOa2E|}+S^z9d#rRSlW)cyDZ~c&OYS?|CR=q+0h#g{Ahps&&gNkgZN5B|M z^G}nns^=+!qc-ff9&V^XjnAEf4tC<-tj(FZ7qE#%VD;&7=i}{2 z)mq+)q+vb8NroSY>Dw%aQiP$j`Jf2ExGSVi+e>w>y8H7iwH#QnQod9NTRdeM({7>U zv!%}0(M2<`_3Dsis`=KVdS=L5aR%($O9+vD1KytDefDIX{K=Zt7gu;x%Y=7B0pl|h z4O!;O{x3Pziufyw86%FF1XYaEPkoeEPFQR`BDD9O_`55+%ScFp1eFr#h@lg^kRs0T z*8|3V7ZcDYktx?;;P>&b_Y6ogh_S#pC!?)~fxbb;V^R*QmffVCC4m1)=#rSWB)22j z-(ds=I*ka~;rmU7I-1P$&bHoMQ}={#*DK&<+3~%s^9r35H8j+-HY{_^*IPM;Z+A3v z9r794c)B*ZZ?pY`s47EMI1!RV04NSGaH18`W*6z9zgP~rQE|`dJXwFHLj0bd-|Z^G z0}HF{$kUK~vLRHR(TjnvHp8BVx=B)#wRR1Vpt0%Th?k3GO{8O_tbfp5D|CHFg{g`P z1aIVmMK+G0J*ouCETzkUWF2$4>MIXyfl_20oEFEGgV;tVMO!&!E#>=Z6?ogHK8O*A zyQgZ`8?+hxS&QWufj_lOPG%izfEMx&VbvG4#Z^h-%xasx0PW$*pVw#uo~})l8|&Wm ztG3i}d2k59Hj{w$vOQ2N15`!qE%3|em61*{Mh7EW2YKEvF4@2)k2z_niOh=|{@bsC zPbvv^MN%Otm!8)7zX*x_els> zn#*bqK|rHu1-0L9)ITf8-Ksj218fs>0JPIv!w;HSbHPnW7Vvk=q@;&*th8A6k-!Q) znT-O%-&PCk<(M@qt*VMP>+LcY?PK{@8pY(5){JAY+^FCVsA3As8tyI|-yhUvJh}Y? zU1~HNg6$kC;d3LcIp>Z<6psy4ta z57zp28=}1|>s(#ijYXDd{<$1uU8>8IPl%H1SywCP7ma;`A%Q4-epi()H)J+OdTyqF zEmeuiJZpx|pYhW+uDg~K4M&7ephW?}0l*dbUV8LRO5RQb4_vZ?iM^A^yUI*mEt)gd zovK87R+jT`*#F&G7b1UmkU5MooP^&jOZYvcd5Eg=V@lkT=9I!VT(MUc+F7IvqIGsF z)-BtM%q$D#Gz+ij;xdIdo-N+DTgd8r_P&4XH9~|B?VvisKmvXbhuK_U4E~is_kV^q zokU#x*FJ^alz3#FaoXk@AZSi1_f7SGJL1vg|7DQFMclbWQix7}-nb*gjT5l@*BBX| zc}K5R)IqGhdhLfbZm}DxMcf_BpbKd&Orb(Nt*gw+MPu%!FHRBmKQnE?zXN$<*(z4p z*4-v-rNviY&~uYbPG7_>qw-l7dsV}uU3TMTi91>=`<3rlBPxyL&s$KNwOi|yq;9uX zIhkOjNG(YwUZy#px`vG#YG1RqR>3F3-MHHCGyanFG@FnF%8IADvmr=|74*yo78B9v zWJzckcRAX2sCM;*r#@9nnG)7z_me4h0RC0LHj^$?1lb#rsa`;7wV&!;9dkMxZfEfW z+9N~n+!Km|&BF_J+N~t9JSEWPja08#keKh zZ4MoVzQi-Y*tihDnroFnH<TgXX6TB>taEQ}Lbl2@p-BX^OM7sv_Ni#rL}3MR ziwO7pzA)=!lB#Mn-l{WP3FN;FNCDM{oLl`tM`ClS28~)Vm3qZFz^&$(A?f&_kchb7 zXf#A7tBSs8eBNnuA2pO(W1X+MXf>OviML*0+Dfx+fZw0QVvWA&-Kg_?UW!nvW}9Vg zyJak4@+e*B!0e38Ldnv3wj{3IrS_a2Wxd5>MEaL@;=_)w$8X397@5-5ZU%a`BrRT{ z>FShS6pmO)Yl*gi$EFj#x4#txOC;eehkZ+J&s~uf8`U5PYe_^mQMeq7t>EQP&RE{% zO29w*&`H4Gxgys#JjbJI(fZTg=mq=)27#zBn~lRV-S*pLvp7`dL^2xj&w}Sc7Oh!_ zplVE0r9gZ2HibR9E^A8js)Y#mVqdC*Axtr@K()l zoEvE0sE@O&V~V4E<(`^0Wwn*VyoZHnK$cPiccycuwjpbVP^V_PBZdv;bz@2|wBZxoH)_zUb^l%a=AxBiHGCNVX-2?h#fCs*7zigdKM8 zG>zidG5B}h($BA7rDmF+d&pMuxIdixdY9LLmo581es`x7;JZt<{MAd3+&{hvgu(&Z z(vpto1Ljo?xt^X3eQf=4Ckb%NjXu37RKCu%x_a%TW2C&ZH>QR6R}xDRMiiJ4f!^o4 zYpfR%0S-~18dpGd0T4<)<8>#8-}~($`fF$SAHr$Rg!aD}dw8vZk$>dDyYx68TMcIo zWS>1oqSzr(NUqw?) zN3R^fo)~Q5jFIYmAVZb~cEzFuUwg5yo*4aXXx4&<|7pdN>(T^A&G&*aej5tW={BOs zbrWsXWt8r8q~M$VO*Ka-+(`>qct8Wd-9U!eA(vRwK6cR6R_;}epjiJUr7$UBMlyJm zIA%G$YAVHQVxi*7shc+^4KLC;E$yObW16p;f>E29Qa(baOFP52?p}-4*{rkh`7kB< zVn}T;LY|qg)XQkB3kGiZc-^-40rBv+k9=CQOeN&Z0QJO2g_F?JbZzMki8AG!et`pA z2n?r@cfG&jTmGdA!yZ*qa_u=LlNu!(p*GIMEUe*ciKN_-k#NzS411=FZO0Z`K?*Uh zjlr=;3DiT6t@~U;9-goQ;%S4@LCNA9{7LP{K^ffapY|#zVUH2U?3>k%s9xXiu*3HK zJ^N_s#cvt%Zy*wUFiBq2x>9Gv_6Z&nbc+jcW&?vURpg58`&*w0ch%aS2~%g`ZhkA! zJKXdEpMno>8bcexzfD3q4h*9aGSKTfNaB&vwOVnX7ta!89P7Z;ToQGWjHWshpaB1W zYDp;9`yDzyVYTpN0zf_Chyk|Sc12g4tsZ;F!;3}RluneEqC_wRXB=~Rkn zVx?MM64vlYb-r$-m~vM76p_5cH!KO>11nO(*Va>n(ov2u48EHIY{SE>FI897PS_VO zG-IL=g4-D=MYBe@Fe{M&q1Nd7TK~d-xga02rzG_mv!YCt4wFl0te$uiGdFU&-GAMk zp6&-S3UYM1U8DUBcgH%+g-k!X*4SwL&)>819z9cJ)Z?`w2rB(zB`WdY05(&0p*q0$ z;fiB~BmJdEGaZ?ra)nTTqhuAAp-=fb1 z4@^?-|M!)qLRwgM?Vb6yTf_9!8Q|NQQHSzWSV&;cJ+*xUZ(8|q(bDbK0XO7q8j^%# zv-#Fxl%@iq&)lAu?iRZuF*HO|hp*&9rY42Z&E#n;Mo$3w-q(Gy2b7W@)GmCwXI$MO z$z&?5*~=_uNj7Sjf;l&fH%f>b-1tU)#76}1yIwCQRs?`->LQ$$aBmDuu8xs+2}QG% zQ_i~%IU{lEm$oM|2yUeB876899_cV#T29LT*q+;9_xYpunWEurfx~*v+W7H!9|uHZ zu*Bfo{=H(KP2q4DR~|n@kquw{>Bi8}bB20_rI@_EulP!rT&w-XwxO6vM#vHV@v7gB zAiZ*YwWQ0fnjbP)QMc+yIv+SK*J18A($*u>2LG&O2-ZiKVK{+Li5CEI`&+qp1$k%< znLv9QPhRbRXv+0w1BMr_E_omV&sF*&#ndcI-Fvcz-X@}=G*cC$sbeMc0@=piXVSW@ zegA{oR*iopx*UlzGN`xfAez&*m*?lGZgOBl{i3cWa>bnTlUGclzs|vWdw7C~2@mL( zuCh?ZnFb6@zpt3F_*D*&lskXLX18m~sa{5m@#z7dX{{(i9I5Fjy_OSjFpq)opC^IiX&UpQ4pAS=hzPEpLLdHf5^0&aWfFK08Ss$c zSv!-wpWNx)hW7Bat?XdGw{9YnS}pW8Sc?G&2(Bn)VMtV8d=`t?Nf3qw5a`-?Wp5-w zh4RiA3tQYk0vjm&5B#I@orwmGFP?A=h6zR-kgUtBn}*lQ1t71t$7(F_iC}98*q&8_ zOix-!CR?9&?}0wAh$~8qm*7XZ0cMK4dY%$~xCQ*@=+W( zdPyby4iZ?q$H}PpCqBaV7h`R=jv_7j0<9}ea~RymFs{SHwS^2S*oAfO+Hy6%_gk{g zT;#m>{k|P#5W4%h%=17zJU|>w;w>|+J|>xjYC(EWvWA8glMp^LR7WY#)d%(G9yA^) zn~AbWO7L_*oe|8B0`l!vo`(DJXL9OdiDoP3S<5%BAe;$Dc6EV5KAV@@?f&x_DrCpUr_s0837AewA5&)bfazKy%x14hxm4SGrKkAyAG< zVA>(lWGJ%>!wJ=r(5mH3O~#e#4`m_0dubNIZC;6BBI3Hsin913niT%-TY|k%Rkcz8%{CVKd_8!2Vx5OCg;ywvpPGP-*w;f z=q3&&TIh5d7-JLwrA$9mJ)AF{u}pv!O@=DL;TQX>g%pO0(y)8KvOwr6KP?1yAUZ8E zkK)i^EA%bQl%rqnB|drh=2pUI?_0Tb?py82e`v^Fj2)vQB<$cp%ul*<)*-|I^)mJ& zd}<$iOR}+eh3Sf*#bE3ODv7?M-*&A8m#%`!?c6&t;YGU2RQot4qJ91I{zE5#BoWYm zcG>pH8WdSNoQC(;e2psdSE=yeA@**Jx4}Z8zYE}BG>0Viun^z$fVX#fAA~c1RH8?R zYJ8fX{(KV2aluviXK4@mqlcDExTJ-%XlC=$tCNB&q}z1~vyJ4pLv7%k(pIP>6=p6d z!9Rjyl<*b~jfa9^AgKn=RhLM|j0pm(^!>Pp2`g%K3UA?OqZY4D>G0f_NAdW*U%Dh` z)Jd8NgyKr=5i}Ewz*3WU_-y>JJcnbY>L3WcbK8Tj^RIrSeYOaH;HJ~o7J?S`PHCTS}}SGUK# z;GLjh)MSGE!RunMo1=|$G<37i z#TT*Qi^Int=k|EcV#O@P#$nYO6F6eP-ou>=ORkBETG{;`9V%G6Gqh5!-_P%gvz{#| zn2(d!<(2<~vhudnabtlW;@E*VG!&{h#-g*V`>;L9iyjJm50USN3au$FM3ERHQ7)gl zjg04eJ$hinRc1RVEXG)Vb7NJK$pW4w6l3Ui=O&ZCJV+uHXD-BgzoOhuC_wPT#?uy# zy)01BQXqjj<`Ho%40^_i5>}qEXdxDJF30-j5t6Dps*H0Pooi93IjS;(-TB|e-|Ut5 zgmaO=6r?crnJ1BEW2=C05@QiC85>ZBhhG6c^06w+Rkh}bL2Z6t{P_P$#QXFt{%?u+ zUK1AooY2(H_oaQ3{t?_?|eSr2aB*vDA>r54oQi`CXUZ z#GZ|HOVpm1ag!+@A|8Der9aa|bo9p*%UPf;s4fS?d`{pwAN9ew^pfS^->Si9ExUqP z8Wh`TU)RlKk3w~t#hQ^fs7e=w zNIYt+7sugl*Be8Yo#e;(@WJIJ8wey1eF`PUe)fe)2iapd9A{FBKll&4kpWU;fADz& z$?a}_5~2i-4CSq5vE z1U_6|Iy%|UMWSfwaD@WKSiOY%{v=vaWB6#_P0aF~^hzlQ+GCR2MdD#g-Bd_UdB2*- zlEY1W);6&jwv80ADdKuQ%KY^~m&-drRoRJv^N){_Ob`L=pg2?z*i4CTGqOQ|L z#f0dMJMXwP3Wnxv(mO-}GoGB?@ZM(Dm(h0)y1|Po6vU^g{Z*89kRIbzl9$S??tubb zS8mtj_rCr2#R3|Um#&F2jaW{aiBISuvpKOpy(^}z&S}uZXJEXdvUPSMG+1=loexHe z-HJgS2Ap^)Z0VeRXFS|vxm9q=>VY%@Tr?g#GJgaPOlB zd6C3;wPgW{+lNXF)gCUBt4PjQDwE08L-X7lx(v|+>PiN)rAC{^lo;<2264`D%WRMt zFupAdhazP}cKLd6{rFMz<~XqBuoaru04M+ zM%dn2v;TO?{`=h1bnqO)CGDVkn>*>L3sKVs^nN(3bqB)H@3}trwA{jvm9bX-a225X z0e^dZ7hnX=k9L1;pPOLnI&9&(4LG-OM=m)^gzr8m=u`cn1=IO$FB8ucTSKQXHS?k7 zX7Si9Yta~>EM<$z*D!~D&fEV0Awe^ef8tdHx7t3k3~+*dqlx}UiaPIBs+_}{*-$FA z=qxgv1~cc$h+>uw2CwUVx5SZq@baeA%06WjSPGR6tgEEasXa>!I!cA#HZny4xzjKw z8#cPVUkN7%^K9gvRY6IcK){u33%KpP|4#WudA7G=Lm{@vPhA}pn}fmSloV1+A$&(03Dg}?74Md zf{3&G^1HE1ZwS5`x+c8Uhd$I*V!BjK*%q0iYAXg+m;&`&4rO5b8e<3*YrRrTRTdd;D0wcd4s8_m{j! z6#yfeq|U^iiSwpcNY4xzHMXv(dT(^A>K-wn_CGT5R)zn_#PvnLW#UK(|05Hpf%>mZ zd|!xf!xj%Vthm{EK*Ahxb#6N1hkDfa2NqyIJvjf-u}m>AWm zsu-N&x>0SVwPRJCh0$Lp0)pmG@sVBc5pP0F4AJhWJxe{)=mJmpj(4Z5y33%6-? zpQNQFHGA-9`EhpJ;E$=B6`b3<>AW$TO%)Y(syHd#Tj&`p18jj$Q*s<=;sDfFn=J>5 z!K9{09d1dJmD6gHMi3^*uB;^U58SJ4<*@aavaFt-CTNLwk!jhR3gE!@3VVw+>BZmf z7j#D=Q?j>kLaK@Ssc=VWaToUP5lQ9v22UmW6fW>!VpT%lA>aD8IQgub6WwfG2g~*o zgkE0;6gskw{g6T8lcEyS;*yS&G#jNkhCpg0q#atQ28Z;t_#%_b^YgjpcaOq`AjxSn zKJV|S$%t-DM3t;|tww&2e3hbm67{%7WkOE&IDwCFvIQmaHLEGTu0G{;kfUv7nWTT* z>bPbmlH61mCH=&}1-FN-!#YQ{?nCp~@yLqG`hU@OkKdJrZJ6Lwv2EL~*k;8%vF%i> zifvZRif!9=#kTF7PQ7oh)iXUGrq`@B|H0W`&VHVKU-$1?=eKA-kTa`LzT$9Jw;bWO zb|mtqyE^+cHA$5-d6A+^0P4+mX3EM}gH{Q37Yg+VM(yZ_0N>UT&dd~AK>jD>27sBe^QRJd^X+W6 zN#vTMg$s^O9mEX=mz8#O*`xY+E-mlEbMqHVesgtsmE(!14()K3!}@HblB^!OOt1;G z@dRp!<~z9WcJCSld>ijK zs<^nFSIQvi{aIdX7iuj}P^z1v=W)o_45bx4Qkx-qytsj+Rv7v$pscv8BcT@}f*lUl z({SZ>l&NObbl?xTe5g1Bb!~!b4n2arM8)< z>p4&w1fdnwV%NGp#bqQ}zi_p$#b3MASl=k?0Q`aRQn3ATH(4)BeBgCp z42{D`qin)EeJ(n33CHXjk`;cmEyCi+4Nvr-p{r)T%Z0#@)$=g{4b`g7(YXAo8`hoX zhcf;0!0GGvk|ul@@GaRMODVKOS^2wzVQwaW&i3cZsLjwT*ESQ(V$r!}1rM|PyEuG) zpV$3T$*N*pF@8Q(0H^K8uhbmMaTM#*i>J{_|T6{xqQ6l^9VZPxitwJc! zld05ofT(30RQAqK8qee$JN*z~bbncH$^87KX3}ex-H%YYJT#VIMaoyjokaz+YH(@> zS-#-ga8JpQ$6;C3i}~T(JX?3hlK-cK<9kc&edwZGs-Q=VW`H zj4RVX=E7O|-#vBP6J$6WwyF)}a?Ikv;D@wM*|v?*t%Cc5f=(h|0l%l4Nz39}JP@_E zJ5rAqi>v$dBjEGH9+SB}Z}KBSxo(kDSy6&=beTmLTLkRv<+S+~&$tw(ixAmG1lf+M z%7g01ny}Q^56O(CoR^s2%60^?zLqf6OWqoR>{UDxwxK+wiGoKJ86`xi(JrM#1HIEl z{~&S>-7>LM{UMJh*s=Ar4mCU6c4LRupTXicw~+CSc0Y_Jvu75_pXbL3>G}zeV5r|% zz3X4h<3yPktvY3!G?gm7M%UFu+a!uN;#|dRW#Ab0s(c3L=)B4c(39TQ6Ns)OcHc-d zR)iU$hf?QC_pP>!lTh{MoTk8jny<^y65EQPFTmiTrV>Lm0B>i{B$C&ynmXW7qKK8+ zgVYkF=!m2A*AGd(EW#0Txr zu`s^^xDZDRwYc^*8xdwAw%8=r_H?|X!oHWCj;n70MsFdB z$Nk#qS-l%kx8EJ+U6Z#ZW{{*`tnb^?ayKJG$StZCpIw7xGDmk7@O^w`^ei>@k@7zj zIop#ImLeP47eyYvRhXcI##s16BMm`Cf^1H1G}ri{;AEpPW0hFr$tc#P;D*h8VJUSH zFzPPd1}yo3>-rk}exl$+*fp$5U@Gwo{$ljYC!)b=);oFgLHW?A4MaSJOk5C+;w&5n7z$l=lW^v`sLsg5-#0zE5s0fP2Wb3%$~X*aYDY0^rY)Qp58NVcSlQ}mtP{6U+n{* zxAlkVKHoq)?R*%!c=j8GZ#a;v5joxwL8=kUnxZjvShBjQuJ=Ejjctz$KMv$-5^)=- zY!er;$0~bZqstBGYWh+R?ZD`4s0VelO)=Ac4TKpW9t4)`vnb z5I3yrBK(|z_(6TdW)5_rpIJex&^{{N`pOk9xAk$TJdJy|(!r6T^~`1K-VWyw)8wqj_p11*3$$edaK&QAhH%jo?2P`7qYP zS*1Utrm6RPUQm;IL%qm~i<8U2Qd?ii#RW1i-I5uEAr;v>cSPx3{>T{pL;{J2TYw@- z6vodo9-H#9tOVx^!{&Bx`wF1+3E3Hg8o0)ZU!eKJ6Op6>;v3TImaezm(`Ny0NYx8` zACua>_CW>=9SUcwuPccnwb(3_DQ(wHBo!x_GmZja^7@N8! zq?ci~$+FTfo0sxtQa$lWG(Gn?`;-21#qjN3Q!j#p)>8~lQXRZ|B?kAV-TlvEnN$gQ z$JsH8wa`OiT53fCjT3_R=c-kRoRG@2A^ zW+xuzEG^SE?1%k~VA+|U0@54OP0rk<u*0c#%ecI zEZRb+3B2v-fzH|G-?vk4yZsjI$ADBwPl9f@FSxgUiC!NBYarIiN4-c%Xx*MyTh{B( z#?Jr?N0Ggt=c_+Ith#t0G?Q2mv*b=N4nyqOL}L`TESSnL8)(HJPF_kU+$-i)Wl%D( z03({@h4uOvnX;cgTX)@?pGbe`e-C0=s@LG%*IEGueUxK?D}tV$&o#iS-Fi`ZOF$jy z_q84L0u@8PSTw1T^!e)qJ?8%flGFSLB!6|;y*J81r_VF>9t0t`^--7~StFro@3Cp~ z;s#eTLNO`;x>5CX%mHB8A*9L83`AM9s}IresLlQMc; z>yvN64FRb=uk{}W@AIiX3qJQXET`Gn{^Z5EgN!^9LcOb8@*@u2&w1e|eZRS?2xICP z{_1p%>z*N{2bWaofNl#AD{g%epESG5_)=*b|(|0-6}72LA!c4?Wl^*bDQQ0LdsaBJ(3t3Jhaxw~XdAdCBM7?j2CsPmnIk z@|(+0O-Y)q$KwIDn}ZD*aQ@3G3Dp|*WnhU;xf9M%Oa+}AcY3uzX zg?BsI_9+;O5d^^Z(>mPJT&4_jAJ)fRZmMKJS4%f?;u623I*XuX({vY)%2x5mxf&qn zk<%QJbx51AWkvv4N{3pf5-St+zkWevjVg@X?DG%NkmQCjP!YlSk4Nql z#!c7r&m*5YrugTPcWjVh(0eu*wHOVX+|7A91T&d)uS9}jjznFWEWm%Vp1N=HTJJ0I?)#QTc;AP*GL(x*tiw9av_TBew}p=$qxK=1LkLv&kVE zP|<%_{(0o|?M7c7c`s^Y%-%ew{}F++NH7k&!kEwPb1hrK;dsd7GYdHhM3VySQ1NOf z#_~SDlgQPG59?8d5YpGt{C|4n2^im#gmp(s$!SW)Ns)UC)Ky%-FNJGGdUZKbDXrIWM)rmUO)2mMe9wI(H&vq~e|xDP55sNzha-Os zcM{>A?!#86%*Hqo!On*~+e`b+MSS3XkmDuB73|RBpm)r?%AxtG)BQB8Zmdkik&F#r9Qg?4u}Ih#NB%_K`^AwL ze5@ir!qD~SqCv}B>PbI@C&HP_{_UT}hpJxa&0g#yM%@LrNEEyXIn^MN%JEz`8{;wQ4^?x{WnlJzRiS7dqVE{Xc(8sSts_(lV zZ%$*_OzTu`kS3K2i7Wo7${vSuRGEY)K@nNhZ)E&<*V~4vAYvQJ6z2gMT&m=M9J$%l z!~U_gc^20uAB6l0)h;1L**}iF3}Q+)U7{h3ix7>f&RnzJy$t68X5oM1$m72_^0pAV zVG?NJkN?5DvRG*DXaf+hFYC?JthJeWzJnVGBo2LhUS3UVv?3Z*Vy*5-Epsd>znwHF2bnWRG7bt_k~s>emlhon zYVSA0uw2YjU`tb(EUF_~i<7XW#cw2-e;9ex7e2pvmZ0i8sJ_)sX>U#l5H z?y)ffy3ZN2YEI{YeAB`Rqq2r<6Bu%gJsQD8RMx*VLfXj7)+Q#`g}k1-XZnKQBe10C z#_dswlh{i)1u_`QeWs}dePDqYkZWK$usx)q=i=|mPf~zsu?QXYUyB?9`=>*lzfmJo zLuDKc1j}KHK!i0i?XFWoG4kz~MeYikIg%U^gFLG-{g|nfIBTy@K8-uD;?=Y(Rr*XV zq>b71UyD4vR(P_njw8W4lv>j4D(7^y@yjB&ikkUgGli)CXORmKQL*GK`OxUrQtnCP zm#LvbFT%D%840g%ZOOz3<+Idc5J)D^jLCD%QU!$oR+Rwv+>Zy5-m*~D-_CW8+_AqrhwF01A}<8uUsiWM z0YMROWMq*NAy%W3Ni1FZcv%GXYEjpjFyEvpVLFjjB4xqN$Hic=&KeDl)#CC~WvRd} z$K(01gg5Xl^+S(^aFj4;X@z0MOP6WNn4^g$MAX-O5owyMjY#fPp=qiH7@=jjVY$vy zAN8kI8KGYUzzCk@Pc$(l)MxI^^5NxbFCOuDWq7DEe6ckN**T;5xTBv2Bhc6#d4Cmh zHF{<`uP;9!lBieiMBctq=b%KYKFuKwB$HQoqNmhbCwkYO)!0vEfYsIs81;}~qG+jA z0fL(KIKSFdO*%UIw$W#Q2^L13Wjr6HLnt+jD9w5>r+{WGQ;xhdO3ObgFg zVY(-*Z{u1R-n(wGkYU%oSn6F4H~E+Zpa%(3ikeEia1`!ORMgjOyTo_Hvpz=-Hs%rZ zE7x~RbQtR!#~9ZM$rHVsUniE1-Yof&$b*GI7Rr<894LpDN-bfWu3h7mC{`OR*f3@O zN#qs}qavRFk;pOsE0Mc?N#x^kZe@TP#9=sX4}^w5Sa7Tc*ukKz<@{h7y6GZz4dW=Q z=>+UDRfDqGAHlm-_|J~3LmSm1h437!5(S7>p0VWV76p}=h23y^>R%GMk@f$S$nOi` zl|QOseB7a3n=a5|)WAkx*os(*X#iWl?vTv$C@OU7~dBkD*^{8h2FU&?Dbzb{n8%I7BxH zXeO-`n7t<)H7t)p|@XWm8>2HNy zwS7xE;X1%-Qj)0o70A@dw~J~CB#Qy6&g*1MLMNHCNqV_|o7#^X{k( zY703P{Q-b4=eh?q#TurP*++PIp5?Le#lIdZz9!zQkxK6Z=nb(jlBcwcB;H%on~Ylx z&UUw)T6QJ>u1eVrzJ%(HNqDIbU|1jyO!gsyf!&v?pHHg5IH_(iLDLE!#U2PJK2>ns zmY?#ne|C)7CifxXINMAv-DH!#ndJUgA`gzemFB+^ywZ?rUC|-+|B}cd4=gjJ=8+@T z)5lwvc5zi&u@s_kS6C)$-P0F&X&)ZIA|1R|BDd#I81pu<;b&@VLgqfrR+5|IY zqG(PCrXs&2a>_oZ)I>TP)IrLwf zhM6__2AYpGD8yt?M7640-jjDD+a?R_ybD3&U_Pd~rA*j{H25j-dOMQNiqK;7Fq6GI zbfLku(7bO6z7^H>>$iO2v)>XhQ0PEgrTDIH+PR4OBrst!6s?u)I53v&gRNYAMlqL5 z5dA`59iXo%yaPUHSzuj~=55ey){%URh<-s29;y`;1N)!hM;?I0G zIu{ur2>Sf7Qx6W**QpESJQ45Z0-lh{3qHw%G}fyR44;}WzU7!tXD?m@1B(MgpSIj? z{T0brSFXNsx9nok~DtE}hh$k!;qqrgftrzf5u61>LBPS$9|^I8Bke z_4gGd$X>%{=s~}lO@Az$_etbxrJY_;RgY`M)&u>Qec>4fB$#QQ8UN{s$@c8R?k3Yt zr}%mjLb+qGhkLe%z`)!q+d5$7o?AV8llNBv6Jqw#77!tu;42UTbh}xZrDn<0?wk;Zq;H_s2Qsrne8 z*fs>P=8lU0Y_?I1|C>qL)@)ubG zd1j47mu1r^`wy3aP+hrPYwWhu-aoh!{Y{YE?T90QE+^XllD(5T(cLq@v!)&o zlW;Y>HTcSMEcGF(eKjt(>+?}K?QHhz!kGy2XwIxv*C-a3`AfClN^BIRI-Q|}%@xv- z#sJ6Qrs-aPb{$7%8>mZIg3mP(=|rDEjG$)ohTQLj-uob&^Tze$5ri_3oK!6%bS&g7 zX$1(yF7nt;SNXZdRa>qiKjFV}RL~7}5O(F-tLRQ^n2e4%+n79WueobhzdD$*iz&jb zKeW;`4#KN9Y^ATQhh*;}Z{|E{e{DxnNiyXQa-y*+P!N-)?CNDHIgxi~c2>3;EG{Ha z;;PBQ!3ENSe_{J0S`NiS{0QH`+!hWF(&7P_BqMCuY&DP6Jy9BJT>*{yrZ^VNsCvwL zw>~-#IYv|aiPbISCw$Vx+OU^or6?B_0|GzlcF`C}XE3As+jc?xoN3Ff8)!M`%KJN^ z+afW7)dunzPTBUC0?~xQY@&TNR8Ua91y(2P600cWsO-kAkW{m(x2CDYI_4pBH9g!p zFa8C(&_s@F&x@iD2X=I!^T$>6WYa7u-!j5jiho?6#ngNp{ zm7jcwtO7M#H^~kuQr)F5%n?K7iud*CfW4qfc3M$E80Nm$Ql)!n1bgBp=L9vXI*b{x z7%cU0a>;oYMm~l`+_2!nC|Mh$5i*Y#K}jcvTDonj!xC!p?Cx{AQNnJU1n)srOmEh? zNSTA%BrqB5Bi9M;9{?F*!;Bo8IF-W=LC^_wrI-;XWXdP1W%D1c$w2WdsF8B~j_Azp zR}TJ(4zcGEGJ-0c?9JTgl3Js$nuFR_1mxhNiL4L=0UcF|e5Turo3U4(4tXEEYv3=L znqDJ|2d6t#VMHe@mJJcA6Ip!0WRcFMXz5~QmUewmo0WDyS5wk4c=2lSrJV`1(vr(G zd*H(C-3+72y-x3jMTjqG%DfceSe`={{>m4V%)B8H&N|>PQXJ;?db{vMM|H_F<2JJ! zQgk|Ok)(;xw!?c|Eo)u-;~Azf1u<{ir??452Cod}+o8t7Y04|C}n~9K0N7iJP|3Y;5m%dryQ^pewyAJRPFz2JsQ?$Bg2~)tzNSSH{S3`~Jd%2$s`=tbkCWmwBr!Wd{wN z&y7tzA1|; z7mKt_mW4)mKKeNFiH&O{PoqDKGlb?x9lqjYN70=@ZM4T;MmNSAVMxR&)wL;QWh@Q$ zh!@}!67luRLAdmjLq3jGN!hR17XDrBjM1_C`E-4p>kIVaat{Ogapn+U91R8X5zK*D zOy?d+fvO>}?RV5KEUHFi*t<`6D}m7>c7E8~hrYAR7o4L}9bu=d!~c2utFmQ-c#MT!x?||uM>6Wg%!FXp!~L!nQQ41gI+*ZH=_7yb9lY z%gSfG&BY2%v<)2tN|2!B1bN^(z-xs}X|bYr``4$X9FN-6bk)xmx7jzZ)CH=gs!s2z zcD8BG5wvCX8d7A?IX;Wr^|_7qA(Qd*8n@m`E1thDX^SJ1UdDQhb1rSMNajvce-i#e zI4Z@Dxb=3W*ktQU86NNML%Sl3$GWo94;jCfw9rm-tn zgiI0jNiU_7nYXE|v27!B*+0A)PN+|@L@=Pk>{9{(+>`j(%HLgG9xT>;_!BVKGNha& z&Q7fFCMnCU3{IOez&YL?Xx72sz6#B)J{qm^Bo%Bgz5*`5=XeRFuY=(8>{Z3$4@=bN z*MZb-zo>$Tuo}{{*R$^hqbsRFhplM@!}gPQYvwBSz?2sqZfzMnH>6cNk3^jEh4#rz zHC(4@+-#_`#+u#L7Lkdh&m{^uFBsSI7#IusW6fbv#etmBEEO$_gSG=WxZ(+)`Ae&I&#QOlt|&XsIEhl`R0vFDN%i7g{{RLe|}Hz7^dp@{55c4 z?~&UW$|N3$*iYP|f zGnehieCm@Bu|km>xiy5sV~k2iU)k5Gp(;rq(~(yvLwg~zhPcc8*2Vw-PbOynK>lAQ zrqu7R_lx6{?Diz1FWj}xLt(M3nIjLm&qtEEgdo!B?0m68wr9`}u51=#z3u{v={oOkZP)hy)DL zUD=X20aH$+tdHwn$9*nPnnc(-#8|9{>>g2ny^#54PJ&Yuj?!nEj0Q>>Z@Y<+qt}+? z31wsvsQ#Z8rhuZ`w(`Zzg+~0WrN5Av@(;;h?j~qBNURH7V;Vw-+XkgZ>t@-KfaZZ1gL=>7Ek_vi^O=y zh;@16G-1-61m`Vx;LYh{Pe^xm(OWI2AlCLs?5!fyS~+UzXvVIz7p`T!^ULT9tR`}w z@g#47l+0yejgCut6`KnJhpUPFxV_d&8?H1j8uy`GpYoMnoKq!~NLD{y&UMXkM)MIZ z3UwXHJ8a6b%lvqur78GpJq~%zWLjPLUW4I_#ZTICD;yVDtaulsv);MQH^e?#Y${|w zh!K4m8Lqnan@*G|+TK~G+BFmphxRL-=Km4|ju+9X#7F; z>@D+N9$V9QX>^~ugX;%v*;dpVRn0@ZF+Ikb@7DL z_D87_@}tPP(^f*A)rox}jCmZZN3=@~As=_%ppjOIu3M(~7*(JVLeb*G(Y6vCQ~vvI z&UdY#d9Kcy{LY^6bSBk98feDgt5(OAmuPkH#E|`2d#dW`O`rGA$+cx7Kc3HgWCJ zvhRqD{S01VHf1B-NP_247fj!jYb!-?a$$PZ2~l;&!iUt6Hw8|L=Ji1LZ2=|KmW6d86o2$1yQzu|n$(rJ5xpYUhw(M=H)O8{2s4*&Bgxk>5sP%UYEh+0ZtkmD85EXX~UOiR+9C(8iD><*XyMN_48{ zAfS{;Ea0?RHTA2ggx%{*5fxtRi_RxCwSI~S1S>AERq1Stl)PnqXw%Det!*p|m1BPz ztK5AJDyI|n@^n5v^{oOf8HBYhCB^$mE$V1KSzYhkFgA!g&>#S>okhpzI|zoM*1$Hnm$y(VJIMv-pv-)vCEnB%_2!P zLPWE_yGzZE8;HA$Vxw)tcErg$zdzMg4xY6r)diD%K-NMozhy?nkfI%gx`hsewm(J_ zn;V{{p(x2G)@*chF0gU}x;{roDPJPVzZ5cN^Fs2bf(jcz)224d&w(g)P} z6i3$(R~jU8d6M$<@ac510Jre@m60Q^(LpLXQ9&HZgCVV8UT6yO`t0$&pM|c?n z^{?W9Y6a!WV)*&$UbGS!XG}PxFM9$W0|LYNq8#HKbm$kg-DjlC4X_l44KpXK3R8N9 z5n@uCX?o!M=tlWytze_2(lT%LGBR+6bgsP&sR(z|5ml z_?Fi9E24*tTj24|=aIaCzUZgDQ+2m7oq zeRBQ<_9ju(Ka)b+}v7H#|65rZ6%IFU^jT&qD|ez6sK$_NU3tA@jfwzsJ|-#n|dVK!YXG z_!F>qc6(IV2l({0=>l}SU9sGMcr^RNro+%aedav6+Dbz0(9_G-By`eJANRE>-HEmUPBQzBA~j8i9u z$YKuR&TK-tfzvq{S>b9;*(oah_(Qr-f8?eLzp;<=6xIdp0X7y;om?VA{)^ zL8lg_Xq_f)eLfz=Llv$w{BM)fNk=nB`oyOmqo-`q?i?-wsY`gzovZ5z6D5wX*&Wad z8K3V%FxyRUE+1WR*mCd4=k}|fFpSl5q|{r_AgoLxCA==~sgE83pm&?y=k;()e_AHk zHg|oRJIiK9PXUo(szKjJL8@sq>*)PTdEH?wU&35)DBb$i;+i5c=e;sRC>TOs*z+je z`>fjDyqn%3DSu4Rs&JTzaZF;I5kF zX2*$hEU#;u+=<{VR;iU$7!gZOaAHqU<4N=-sLDl;5VjkYZkHSzNX&D~B|yNSNvPy8 z9}Bw<@I=V|*vn!z?EwIWRr6xk2^N#Xqpm_i!)$fD~F07>z$S444r>@-H9B z_aZyG4r`|}9^&PFj6Fk?{y=9ZIX%BW^?Zi=tL>D!Hzc>fjZ=L^gF^NglvYXyn=_qi z8FPZ~&Oh zW;Q3FgU!RA6h=m;kH%hq9@A!=siHDj0IkIJzZsJeNu=)t2}G3VP^@?iZo**ABOY3x zlXOH75!aEwMHl%TADxr`h1RM(i$eN(x{{q}&H8!~xeo~B@|sr1r@~ z7U5vwC;@0LNhDk$AB>!TsOYx4{t^+6Mrq6WQ(>?qKhQdIv`2n8-^Nr#w`9T>F4`yq znx|#hOniSNjT6~F&=O3J)%w@X5|ai)^3$#rQpgY!lssHC&Gx6Ie&X!EG1Tc^TrKjT zCL3i_{$2^H*2*a5YtRg{EH7=Oz;Ks${PgiQ`>(7se561oeNbPFvY*x8z!hz~6yg*e z{Y*?|M-&-e!dQo%tPARmn<7xMzmpJ;lILL=SviJp+x(W=G3vyG?}SypcRDfY@!?K5 zA}z@vyEsE8pe8F55vP#KqkiBK$M}OXAT?sOHcfKys77y+4NEeZ{1-x)M)I5a$K2=H z?=c9HE%RKkEgFLHz7g2;%^yj5G))`6#*)%UCi6&O;{}Hy@l?e(U7C*hly*>Y=*8-> z&M|Lons#yK}0o*0Wxtm}0@b!HqV2 z$B~mUY%FPb(Vp`c%{i8Kg+)+>AWS6t{?2Mxc@*RF09oave z*`dk<+*^%)k2C+Y@J+`fFP+76p(-pzhW_Q5>;y42@Rx_3K^Sq9k+n=L12cL^!k(dr7PU%Bb7`4W^m1l=r+Hp6etM+$Q^xqCp&r7Sh(`rz(JZNn= zw2D%?0HBH{!~S4TFI8Mm1{0KDYk5-(OX`(avse(8yNVY>vu#LDym#vzw*rw9wqYow zJX{{33C&ItF57CP(;%y``OET+Te6oPE2{~{t&m~9>;A=NV*3>By*_|6@{s1>jeLm?i8;c(=kFq}UbPjRwM`kNhV8d` zd!iT_ravJC^U=a$A}5`)im+C2QdJ9Vk=wZFu``b>=T7@_EDHAgNg-6qTlg}530*Se z(tME==lS__0NIH!L5+D=dMq2=cWp$CcW7cVbVLj2q<;(Guc5&&fyIbU z4_e`v(>DlVisScMK8xGzMG0RupW>k$jn+!#XA$`zhX6_XrzRcJ)&#;b;FC4thf z8Q2Fb4QGZd`3!&ZWGN_pr({u=uK|i3lNX8( zPalQqmBrM`59%_*l-s&ohh@;}ajunqG9t4@$GclHoj()1Y`gr3(LaB!+#S zOo83{sBas^J%&Zreb63LNGm>ts)VypboP5Fg&i!2XzN|6ZhTUU@mjo_d(`W$HhPNCr_>?ap-w%b&i za0f2)6DD)@pIStD2)%7EClF#}Gk$=l(zPT=x1hKBHgKoX?+SdIr#zpp$xpo#=&=+A zoEJP6^yvX^4MK@Tk65@7l8`W1Cg@TxXl6eAA=V2W2sI#`A_Zw~Rrcq1?e>y}?CrhF z-MUD$A0>8oBK~YWt4O~g@o({5s&O^&3xp1XF^Z{ zbR1Wn3BHa0O~7PJAHUzn-0T~MQm%l#^S$%0Z{43eJN<9nU-xB; zBZ~pg!SrS!LAGWIwJ|$JPVj233T_tJ~3()jfSwB-L4ZDu%JR;NkaGR z6GBjWS}$R0-Mxy2>_lqve2$;gRL36Fwq!%wMEa&X6m;N@n=!J@dJ)5t^k~Lxxb^0e zl9Hi;1z1dwP)vIG_x%ncG(`%p_b>C;$16enME5`RdF`P#ngffomxatp@ioT&vc}J9 zf31cb$InHv?Uca?k5X{|q%6=x48K|~a9uw2w|pk1yh?uA3?x`i3-<`g@h+COE{A`MQ%Jc?>Runv9Gt z?jR2S6-ohVAL*7)?I29&{dIcCmsODMYCYXFh1>7}(Kh#&p>|-VF4R7~K6kR=!p+YU zlADKa7wv7nx`DE2+mNM2IJ!NwTiQ%s_s9KUhG|@PWef3yp$vIj-l4QM%crt!^{6mv z0Hd+=l?}?iR&R^Q+R=6oZWR}H6PQ~tBG5LkaI~uzwr}mb)(j4kfYYyxlUe|(I1)u* z2Aa-(GIj#?#B9hs4;6f?o6-!+vOO;e52VH&3%FJ@X!9b2Eo`eTTxtuv16-Dm&x|d* z&?eUOe8NDZFT+In?ZEP76ZtdyUeU{-yCpw{(Dh(q0oKfJ2F&kb)HoDb$Lc9+?gag* z%_W^0D=TTV=(3{VU}1~^nj7$BbF7s6cCjff}#H{qBl7H~s`x{klPxiJ>m zwJRldVj4Lgkrg0oOkJG>@*O@3C zZc&8BUweINwK@m3B@G@_o{&WB3G3-c5K5^VDEu`e;Lx^xcQsJMt?iy3ym~AuL(U>b z1o=b%HafQ|ewLoLa3fMtwm70{zp+O!>iEMRBF;qe0sWu8Lx305g8XC5;4g{!7rle{ zMeisde58soaEctcS{|!F@ z?;DGTv!qeO?gC$Ho5CkC^iPOTJGM;N1=|Dk+-a#iXt@M1YY*)eIAZNieZ$sgYWNK^ zPy-n1N$o0E*%OE>#5yBG)T%y1BM@WUNO4oM2TJ&U$SUD>)ML^wE+6XXlr`68mM1Hzzv_MNqDY|0QOFs>|drb1(=OFKfKH&@g*V<|NT<&_hd(@W6n}@ z(a&9vRYKLHqlJ;jy_HrdHZYGF*)C388u_gJHL(~ z#<4JaC8d15Cw`=X{Q$b6`=oww$cU1*ycO4B=?o4B>_dqpK`%VM`;X+kbl#g!(J!gcU=o zGRb?;kaL2A&*H-;orvWze4OG;T1yCfC~dk~+hCCY{x(HJo?Y`c5BKwl)dqn7FoR;Aj}-rb+D@{LBiV{;e^9H9t3-g&o{#=Il9m@cC<3bnc0dc~|}KC2bYz z>UFfHzD3uuY5ri-7)21|``zbVm*j{R>JLok5#}Jm@IANe^e{$?@Kw$z>xf!}{glzD z1;!>-oYaNB4k;o+Vv_Fe>HoprKLz&^xQ)7Ck`)^(wryv{wr$(CZD+-{wPM@0ZJWQz z|J!@^?3p=L=jxoAsrqjEy8EKLtKWX!=iz$kA-jDTq5L0u8TM9aT?c@dyR&=`fQRFp zZ_uwZX(l$(=|CU{-U6t}Y}Sb+paLAr@jq4>s-Y?N&Xb)oVAQbf@0J#!Z>&;<=ZGXH zSQ%>Y1+U9;yXG(jaVsp5;r}U>L4>X#|Hz?WGL?9i@U{6RZi10Zg0k#hru{Q&YcS6# zy9keF?hv_gkc+oqZlz5(Mt2-7bg&sP4lakt0pwRvck3rX!Aua|`HfGL0`FTLrC=Zf zo_0=H?kYDe2?q|hn~pK;u;m&w=pLN_8v+a*m(O3C$6(Y(>bEDY$}64)*^JZ{@=xm+ zlL+y}$bV>P4|7~_eMj-;6LCx{BE;U;+Y@v9CHL~2=;8FvP0rhstWQ#GiRx^(@tYr} zOtEHP6&`tvhQPFue5v*m*Mc*jN$ri+;4N(AW2bCe`^+-iYP=Cggqo2a+Z(cL)!wWuaLSeocg0|~ zEZl^q@1`Y9<)PGcyMmm)wtKISoqJbp?>2MPjjW`C1oLV!l0`8G?#6OCGs2dR0R^aEQlixFx~l6kq0Dkw-<}`PM3zmpWv0fp*>U>S zC3TKswW7;=qLXogeHdXyt&RiuIWqXCFka=fU(NpHC(^_uqZhGgpJzB0*H_o&i7o|ad4K?J|y^UYIGVVvOez5lpg zC}lwOa$oOX;vAarkW`Iz{CrGVI6o3+xzQUBUaQw^KCB~;c9?Rh&7>?2t_3z|?U;;G zJX_KmC55gv1|Iag(AfUAK^xh!xw)=PttpMQt3inXy;9ZC(GIYjXO@jvVL zwXil41v#c%GVMXqX^OV#jN0X7cR$?fuwpEfVa;UFb_|={yl0L)IL&KNU+h#6LvUDB z7M6zwXtDl3EiY4;zEfSu)FNrs!=3^?+K1S(Qp_dm|6ht1b(k_yZ?M>P)xwIdC8n}* zc-C(l=>VjV=ptj}{gY(eGP3iv-yzn0S zb!H+|iCtx?JGvp$+&zykeoNum`JW7S7VD;JFLy*Qn2T3|Hf~3!uoRHudIs-GoiywC z2IswiS2ST?M7Z{AC$Y#5U6X|s@PcY%(e>O{6*pZWXN>sKHF!*i9e&Y~$*q6Mi+G0L z5QDi)nZ%M&I1fcK;)DEs{<_X;l$2?G?|tFLcIUZh2a+j7#mm8NmC`@7NTu?h_|ahu zarmH`EwzBZZ1?CF1M;u#dr`+^-sa?8YxD+4gFCjaX;Tk+hAe)UqQA6LH3nzbN1SR@ zHUc2yo!bDbqW#9j_tNpLEqF#A^zpG^nPVmLt;9c{OhnxD7(F!JQZo3-;Yz?3jUUke4vB#Ph zKF4R%^UZodc5jT@0pV8TIXg?#$by&9<2y>9n-Ow+d^}4k#Un(CHJeP$pvH*tl}#Gb zNbOTX3-(U;7{QossHa;SLYLLbl;cF#X_zoxJk@y$fwrRU_D{>9c^B|4&e2$K^n@;- zFnFG{Ih2bQQRP)c*~Z!-E@wdenTE*7o7dn`e&1eimH)!rE7Ee1vn^5~D}5>!0`z2m zwdhLcsoeR!2YG(h4h$3qL44fM@xB-V=D1G;TWrghU`1)LdWGgaW9UMK;yRcb;)Np35V<7h`Rw}tyA-AZ_+jCA*~-X7f)xpGC2Kkf)P})@IY`Bl zPt!9gRX&qtML){7XTyB$2}40i8Qb%M+8>C-*JwL0rk9)arzQ!j6_xVXKWnuOcSa9; zT%@aq)@&sermkP;a22ZF%<(g+8oAV&zmomNE2hW48q0FY9=8KhKAVm@{87X#H0A;h zR^ombuq)+tx2k8&`(GVib7h}xQiWxRSKr>xHI3263J@&wH5|ygY$|d~J-ER|i8-s> z54g<5n_PHbf}slqk2-M6t>+lcxaUQ(-&u{e1-{yB%f-Hk5PWH=ZhDRz&gIBDzL+Ol z)D%ue_o{6cm$7|E^QdS;6WGG-t&9<@UV=(VyK?kiRj^)J&C-Ol~TZ7Y!Az;vEI*#UZH$VE(NCyw=m*eZ^dYxhjT41vN|;7i#<5~-<|P8JVf zo2+~$H!m07>xnW510WM0V)Y8{PRA*msHhzZ+Ug{No>W;%BbJ?g zuFP)_)eX=8pEek3%r-1b;lD>PyM#Lx;0iGob($$ftwNTK~&t>*c2?b)baM{b@_mirEdXFW~s z^|;Z~^X+26n4C7d?@%aH+4Oexd7?KhtJ~w<#2&FAPG^(XCf4p>kp447l=&n}In1yp zG->a*{>y}wU@HlZTZu1oL$+E@6myJ}i;SZe-D%w)c9f&Ng=FY)1) zC+5gYjzH++(VOC)lWwec%{WHyingN3x)RH}!4xWGQNqk9A={V})PsR?g#c}`&{M9! z;9|mU#LUn4Yg)I_;JR$FOrWMVlNh$Nk%s685s{i`NvvsHCp=>P_iOL{5Bt^=)zf`t znJxYHwB7oyyOC?c$q04ZcGYQFbK=9rqJ4$>lFy0qpY!Pl&0%3e7i@_ZnTBO*)vD)A zKYA?3Qz2B~CSC_qg{?$3%#{fdO&#NExvjA~wzg8@7|w{Q>PWH#Yz?Gli8hT)c$C7i zIc&D82EJw0kUQOp{5+dIQH6wtX0M^XfQ7k#<+@u##2)FMI`r}#tC~v#B^h2vst!K= zODTCgU7XG?d@Fud48l5ALxn*Tcjqknt(e=`DOR|u%i zM6yr$ez{yr;`+YX{r_78CjXnJexXVIPTdm-D~^^nj-A+amC*qDB-VZuZ9(_|%x_paKD zRSyThd*>{k_cZ*EJ`ZmocSg!5*6S}&o+11%wy9dRr8jTqJ*^mV;*La_gV)CxZk<*Y zrW{1fYI)aoHPaK)L)^5!TDMU^#heNy5xhbr4{G7mQyOf-r$a8cVXjduVWvqYYUK0k zjtdfoW(cyA=BaZ!xmhiP2oZ^$G(CtTG~-;vX87SEN%5BkX>o|85!)EFPECeBt$(=| z=RYrKnU~wPpc%L$ziB5I@J-F{m$-MCH@`=FfD?IcEg|4Xvba+ZSEuTM_sP$BSNZbY zH5|hinj3;f&LSHQupKWtu2k|D!918cY-Nchr<5?V7>tBnEuj;|rwJJGVLEr0-bNmnsQJpJQPzEkt++6o177|601~g%f>N{K7$Dw!fo+ zq~{(7SlA%+%NNO-;`DepXMG9$fnJ9D=VA84=kURf@YYlE{^?xCrB_FdL9>C(hG9Lv@)VniIAu$E%HQTSbEZPxA+gqob&yta(54(F=>m zSTn>tJS1gUkIz?}IN=DpfuuR<6ZpUE*@)E=7b?g`|(*q`}`Hs9#f}8CbLdtf`|(1EyH{nLh4HS`(bHWkVDLwW70U{67jGw zPA{5P$jLu#NYn%7MH{h0OqugUhLll@L>Dxr5UFHMaEp+II`RFyCz#g6LW8@z8b6Dc zeR%>yN8XdSyqXIQ1!_Zn(ymM==XZ1vis~e>l4z;+y@?5)3oM#J&DS`e3~Xm*1gOjb z+C$0mA+m%_#t}G%)6lp2qtflno)MU|m=Ja^;$TsD)fhFajBA}uc-ngt8nVy({$Nan ze=u2tdZM{E(3E@qM23AAo zAy3H5`H{w=il1}w-jYC}#iY6)p@dYiJz4 z*G>XE>Vd*>4(j`zAbZfBiG7k!L6-b{zfalxrljDr9gl z5)J_ypElceZ>16sVU%@ETg&Ki2wR0trLx8lf9zj7hAaiP=wN~J46Fnm*CE9|g}J`y z5~mCj7-X{O50|Z#e*1};-w2jkfLVCF;V;A#Jj~?tJC$lGE zMhjqUb`zi)pxO})vEeUxB@K{!6H$o)s0J7@Z<$JdRRbvp4PlDJbu! z@^d=F0YKEiAF;1eTdM9WhsrMeHDlEOsf|FK9AwC@WY0y>s@@_NOs zaEKjcc2}3%bW^6eOc8g1nzhgptjc)<$jI00=-O1M|C0S-wIn#d4CgsQjZqsdlKV}6 zP>dW{F)3bB^sPFK{9*fAy|myanuPQlm)qmnFj&a)kXudo48IqJ7=gD@pmuEZMJ#cN zZ=y;_O5;mXcFnWO)u>r?Hg@&Xu-3K@?<^+CdNWgjkj}HV!5$tV3Ur7kW2BE}z1$E+ z=XpZWX=r|6=~2>U>7N?Kr3vxkV4txyx+BeGp?I8DqW{!rmW+w$An1|C;8f1?c$ly~ zKL$}*$U9$6^?9Q3CTNYeJeKCMkYMj%G|oZyG#R)!Sqd28P{dWw#9zrhBhW&}K^E%cB@hkWkI)x;9Q1(-I4PJ=4IMziS zSzW6i1$QJf|DC}!m+Egp&Ws=e3Uw^T*1ME6oje`6(K0YiR*-{~lwdLqPhfD|EbBb= zD;OJUV?lxFw&J*btl-GjtV7D+&$OiCY#?&S;sRpf$E8Ld@)hI94fL@_aEIrdY7Rvz zUg+7>5L*0soY)X+Lu>iQx~dqG9m4U1wU5$v*DIiqKrCWsJ{gU^s_by5gqabE$x;_R zNwummcjPe36i~tDep`XlnItB(um}ej2|CC6n|T^Vs@TshM>zj}8c=cMFouoO`EIg8 zbh%jF#Suay!pM4aX={-Y1=6O5B$-F3e1lY1Wi+2ypEn}Cg+42wKP#Mg;CkJlpH#|c zy1yXv*1dBH^7p?-M+*}2HSdv3qHk72%|@(IiMh=L`4c4s=?540k^9z|68eWcX)!!e zi%yT`$NWDWNoGXk*~QjZ2)X1%l|v#W;{_GYT_UNZiNh1(m;U7njbBy%BwbMjBdP0Q z0u^IHrr(RdR+v*@1iNSWA-a{gQbQ6@S-R3Ggp{m1zJ_BHr^ib0!BfSi;E3g94}9wP zg=IG32`FW&@lJ4FT)Kq9RjghMI(f&<28ous9)CHILRI3099^!H=v{JMW4<4Sl*zOTKe9@8EhLdi9XL*E64oIYxnkybf#yZj5nWhjKn z*Hr%olO%g^gcs$__|uY0jE>I#i@!0kqF0gu_}qNF(Ol~S3-k~!ka5N2Tm~6nYh>{4 zR0VAYvj1T;Q!Vz>CNmv%OM=TF45ptVV~z8SmPOYPnvk%U9BmY*(b87Ot{N;ZcU&oW zNu&*j4uY{-bDu@e#I-PFV8?nY*|2k=+Bp|&yPl>-|3m1^OY@P90P}ro(JYw5ox-epTIHS9u_PVAX<* z1}Dwt`SasW7E?z>PFxd_T)X;2V8-#qsatt1{@te!Ue$Oz1^axC217en_M4QuJK=Z8j zJS%x|<H3&!w$qZ4bvIHmx)~Sczgyy~D;b z;B#oxI^H`N4HlpIN6++uzqncO4d%@ zCA8&LWtp0Fkdhr%ue7}1i)iuMpBE^lVf06uZO0LxDR@+}&aB5zW{;?%$Ieh1*HO%FpI)G*T za%ij@=mqkrK=pc14#Zh{feDnHeKYcyJ>;p3`K)-Cjp3_x)GEFe8phFTbm=S|<;5&R z#F4k$vDhp0R-E~m`9M{)WlIy!^P9zrP;b%k}J$uWJ3 zmGt@QCEC<|CTevKmMYtx%W6!QS011_G7;FT0sF~_G=z!qPF5ToABnX}rw>^Sdf{N5 z0W}`sUItDhe@SLR9DY>w4iQ`-p__cD%t|HZT_cnt8%cVGJrp2LltKU2%PV31c)V5IU`gN}~&iP0>v>pgHY zabUVk!L#n&^Xp-npTmf%Lx+(DslCqO{05nG7Id7VW{!oRq{ZsC?QbpBbB z9*pFtL!R;aq?-<_O9<&FThvU>lo@)`%GcDBIr5OSA&<&Kif#_$m z3h{99Rc-^g|8P|t0ODftw*7X-{(qK%*=*2l8(l+`BXRce^mKji6PRr=emMWK%Wh}a zHu~ZHch7b$4!(AFcL@O>KVQr&lCUS3v9IsEv-E&n@avTW{2fe@U_nJbas=)fX9QrhlrDng8V6O)8HwM>3fNr9D39F9|B#Gs<7yjj z6Hx;>>_&>U=!Xu7QzIC$V$wOH{@Vfb-wv4ncEJ3fIbdr4+X3_64w(OI9WYBj0L=UU z3xH|=-vO8(0%jAiJ!EzQXnbkjve_8z&&B_S=5}uX_xU&Za8R~j;SsO)z}Cku<9+q) z`fBVff&jUpwM0BM@6z&O)2V02&%ZR+-`|L2XE_hI%{x(D5{U(Qe zsSp*=73w36NM1!mXce#2mP|_PTcq38AGo~x*-U`zztNon5@|b*w%)}iL&hx~F`r7> z=s>Ln2AY_XT?Et@1kn6E_lA>jGGeq+_IoR4f8~<92&McOi1EjfaCzwXcCOBQzA?gFd;s;y^!Q87fAC>oIy}tPU0H zW`Bj{ZI)+@u9%TC7L@z850{;ro?B1b>ia+i#z}ybGDS-chS?zWxmq=w$s~e*G4vlA zYvUc4Y~NAs`9LgF%W$)G8=Z1FgJk>J<}`3Smh^S+#H-eMq1VupFO5Ij(nY}0xr3WZ zsd|A?w`6h9rXaep^uO5Avg4P5kJcvxu-(}btSUX={d+1c5j}N1*AX`L?=4GpG=P%$MLIFf(HS=e?{6?ROIHcw0fC;`G}ty9W3g0 z{cO=Ru$~V#vsN*H=CaUZnYY|N*3_*va9s-NS8Bq$q8hVWK5S#EH9KZMUkqqwHBypz z_R>+|yb;c>C{8T|A&0Wz&{1-DifPw?W+`AP>*0;c&+PQ!&JPpsb>B&4ac4R_?q)E> zZ8cq_pc29S{L|j9zs92zGxu~|7pre#4X|sWZHaBIORJZpp0FOcW)i(X`E_Pa%wxuH zgqXtV9^@cbA%gj7Oj8&GsW~yT5*b_IO(+bU+e*rroV-xAM@ee5p*+LJv%*|npB;+K ztSPI9X-B560;UwV1zN{hE7gZ`iL7>9?xJ=cg2v<0vOta%%6ag)K;F_g&mf}F zyFyy?`K9!`20S=HV?_rSQ)aT8kY!@CBeWVzb-a>CnZsYW{}T zOBHjp+sYlkime~(kD9>b*7I~$dz)7U2YROI&?Zwf7&Wg0_LO#m(V10tjc>L4KPU#> zd66mqj;9GFxMF(Ng9%&>M8!Jd@b0Fo%jsfc^@@-W2^mU>U*y-yO=#2;14yIrh~66v zQdAUbeRPJgeIii=?AUEn9ShC6-4kd^@D1{SihmSNd(B2*LzRr{f6>6-zb-@eOV^qY}i_j4OkeU*}ZgNZsWUICU06q`ZO zQWN;0bRGbo+lIUEhdj5}$L|LLUjDdsw)$STefqlR@%e9W<_SjB2Z|R8utnmXu)SY!%_JNx&qPtqA_r;kp^at zOjm!q*{2OcEjKR=*81uGEnBsq;-RWz>v7IB=l_J3s{SjqwA8*+5mh_* zW)On|wRjZUXD)6;>G#+D&3724cZ$syX0PpBB2KJCfr)_UM=B%fhZ{c-6m$Xs|FhrW zHW?Z^feGg)zW?Tt(XWIC`$qMG>%=k9$K4P^pT+hs*O8vPVaj;<)aEHf@_zy~pR)f2 z)EGK~7f%?xNZK8G4GR|HS)~6LP{V?Vj(#&{_<21L#w=wVjA|Rg;}M@)o;93? z_6Fd6->KjXNIe3aJ$bQy-G~6*l)h^r-ne7G#Tuhw*4m5Jm@(@yfaoO-Si3Fm68S2` z;EokmcS$Yp-tBb#35D@lo~=@k!6qOEQ}PAhQ*~fmjt3wnQj8>uq^S~I+`GzNK;fAl ziT-wWTqq(qh=<5=7S${4JDCZ0&#zPqr$Q4p5lEM+rs`1rWR{*}wkDV?;B4~tlyi+R zF6zN|*_P))Iyf&4zgp+-Qx`{SfV4a)=0U7TO+8fXrx?ydW;5W@fUHn{h#F~*@mbA_ zFSc=yyOy%Q-eEe%0~H_-zCKsY0vd|a!6U3@jU1x5(jKODQdeR^w(~fa*?U;#b2o0- z#2a|Tmf@CCc0ORczeemXv7`6W+Vt8R0^n_-6n|#Nh3Y`7tV{K? zH%k|m%J116qEN5QJ_EhR zrWZ?P0)v1+Kgt zOJ($I!}YovO&ePrm!p8kG~73EEk*a#LV5aOpuR7qg{6&moQ9;kKosW7T$TfUjei}K zsyu`>zjd@S+L!hu+hJa$9jln`Z=`5}f@A-2GlwiPu#aYjJCNgy7Ym}r_i>Y-x{?HNiu~P^tIN?&Gkei<;jG^OhBgXte4&t(yq$L=J z?f|OUpx<1)Gx+h(FgMn=OG!Dc;~UMf=8C4=FEJY@%p}xk^uZ4?lLh8yy(++ty78b84PS(y3P#LPsWUE}3Vb55hqJXW~{NAFAYSwM^Zrl3Q zVvHBTt6<1m3ZRZS>h)McAJr?GN5jg4cA;K{a_gf@<$%HXdl7FlFFvNh6FjwTbm z;b@WpsB8Qd?MW5k-|!bx2xwsJG4$K9G}ltAdA^m@CDC2MUc}LwFjZG`taV3KgV1gR zow}ORuNZLvRQn?yEhne@iL2|0A!|*ORLD8Hf7Ve%JN7|6`O1XGglT6d1T6 z*j)lyDk%g@vp%_@bico=uXj8xdOOlFBzKw3S(Hf~k+&tOs$)U5{yCboegvbkf$sYu zt-W5~@0_Cor;Yj1>+#l(`OP5Y9`=O2aYufZ_5-kxd@_yJu)`O%`Wq#r^6*ML=4pJc z%G#bS*b!M)c->h|V>zUxwPC!wT;*;;g?2MY53hcbysiki-^*n(J&->Q_?_i&?MBIb zzs26>3ifYaT{Gc{#4^^#xpzjZ^yx>Jo^V!bnQDAJ`I{C&=RT&R1B9Wb=Dh4<$=amF z6inw3jg1YV%V_P)6v#j6do%OWTj(B|MFLm|qhUs{sQuakW|sZ`YBt9?)pT7u#$IIa za}T*2W)ubN(?3`IH2eS#_V%-Wi2%HPr=A!5B^Ku?GlCcv7gdM4A|5_*r&PgUyOwD) zWx+Aj)u}}1Xg&X{CgW)9BR-QW+4~KRZq38lo>F2&N(hsL-}(-FVN6Kr#;&Hn zYAL%j%G%7KYu14*SOxwO*dEM?CZCKIm+R6b*JRSTgDXsw7Tx^CO?i35$YHoxU$1uRe1kq;XSs+S!`z0f-z5Db@_xfD zv#z$lSZ7vpwP58zJX|)4M2n%3_-V>)IC!(4u3AZu4}DuhBvyr3fiKpnpCXw+J$O<} zWBM0}R_c(v{m+i6B+Cy^UaO3#*Ph_8uW@}#r}YL6o?23-E&&i}%6fN2^rlNW1L)cfV< zUsU>9xIy;jQpuDXMdodUSZ=IVVS$sKJ-!iwOSAtkVR*Hy)pS*}g{z7=nf_6(Kn)-vmMQY7eg5m;?58kd5`FHM}^cFDoE}&~b zHwn<=eh=UtJ(0QvxO~2Am4Gad&wSlbklQZZ?AW+}GJerSbCf7ZZz7uP@u^7;xuP;?GPXhh;p1#&Vz(U3Ws$b14T(I9HivDTZ> zf-an>ONV}Pn%D@A9rKll#0j3n#=mHDtU&}82VNP$(`XW_q^b*BV+0Y%FpO;P09WrS z>TI%EC4XTi@Q}B^H?MZ|ygI)vrn6`rVNH}q_2THoF^)Fx=nD&A03x@zevfgTFaQ-j zX}XSmzH>R>Z0CspQ5HlT1dxVoo!lMc=^DNQwdoo?>7kwl?sjKaz{DDnU$IB>GzX`T zS3fS_#6nMFp8Y3Vi}iG~;O5%zWR zvGlOx*S6!=4!}QbTq?V`;t;DKD{KEDYAk<iEDF~`D zTNfl+KT?_g0vwRXf1DD=toHTdk5WQ9^M>=i zL2|L9Vo{oAI^dQwThi<=%1E=aQ+|T6GZKSPF&-6{IVL)eHUTUsL6C$JnD~1#M1_6A zmy#MkQW>tb_Sfbc4nAm^lv&V}84~?fi3u6|6k#wG)?Sev2dt4ZI8`yT2>llCUH8LM zXHjI+DK!uR1(j+o#C6b$253UN!9z0s>^o=GTMA=nXZOF21D{eo-sJ4YB8c!$s|f^+ zP*Xywb@E6WW{lkY84PUv#L2F}&>yT)dl)JFk1u-fJUu5b*z(#_qOf0cSgE9qRG&um zNtkMJZk+!8f(|MTdHyh|uuGXGd(dUgE4yLM5<+0*Wl)D)1;G`kzloGmM@da-AYekP zivAR}5-8I#|q(^I!688E+1)E#i(!}g;>DBNdf zjWOe<(!^Pou%nHrCQgeT?99?RPhu&$k)7)cXzfS|cUjQ6C@mpIj7Jc9y9ny3iEITr>DU9fl! z*yB(~+;BvB(w)?aZSuks>i84W!cdqCRspgOBhaCU%z-e2dk|5}7k^yJMvQZ&S)Py2 zXjUYfV@azL6ILAn3ci;(`Evph}^g*@+0Y@1JuZF$^Ht~*|b){5&9d%vL_$o-Otj3G=i)bw+-Z` zH0tj_O`Qr=IOvmE96`XR4;N+LCO&uG8K#~9K@dxNr9KM7EKDR74K479f)zKhPOVbNxN8b-=3~T_MnpOG+I5S05*z?Y`+c7|5Dx zl+ze2W`BIyyw;}e#M7y1LjygQJ{N@iGIUifg2+B~=)p25Dv*qDN+@<^us#dZ9L=s- z(Ho>b(nvyhm2gRYGfF-&8a2>l9J`usZ`=ivU*1jK{nxqvIXws0R2rHb9Yb56(vX9Dr7Fxip3r$Ed4r?NDUTZ>EZwIc2&FrHj z$pljHbwaO);2;r`aRN>8^-JCGa7YbrR*T7bD_{5tq2YX1P`M{X>mOFdRlTMVh8T{o zgDQJooU@n%*ST!DKKfRR1w&An2*^I&6pE#g^TXjSWz88YwN6MKVkw1CqxD0Vgs#s1348j4O;*sO6x85h3r8qu+)Zl&^(!a~YJEZe=vKlwA~-=Y5^@h1 z!tWzf3*z4TB4w7~@V54nqr+5KzYiLRWuR&9MpPlmAyMG@hS{TE8GX(1u~N8ABs2#E zV{YM&CdY5(^EK&t^889nTbWQ3=?=&a^bKniv9Cq8Qg@&-q6ZZ6%KI>L(4}A{d{I&2 z+GNEV;)V%PPH9E?$PR}$p%y@(rhWYT6#iSHNzb&H-M+rB?cMO{FbY*7BZI2OI7@;s zf6ml*5VSuC~ytv%q^q09tCUzgpa`hl8HY*XJlrwKojLTa8(0I(%1#9jj zc|>0#RuU3 zjuo*m29qu|DmITaYFXh>nCDn)8UXTGoMgB^rm3bE%_*zj0wP%}XFP~LXseNfY_pW= zq^OK*z^s>-p}*qScIdAVF<@+8E=TWxemCuGLo7TeCc`l#%vagV2swRfhjC6NY7{D$ zllX(ZGuthq9e+y#M~X6|;V)X-Mr1&GP~L(2=uepqGuK~7FVB>N$}E4cu+;{@TdM;? z2C7I=CRGiDlUGwp4b3dD(vNo&AJ#r8ccc95fa?D|$%D26-X9T~=~9!QWL6EdgTN*c z64%R!wL*J>jG=1?J5JJ1j(1BS&?2c~rEoAP-%Q!=FDW2p97RC@U=U&4>b9P4_hk-ep-rrE)3ohd*iHdT2cVUMjqiNL?JZ6H! zt;j8gkdS;VZbg|DW&jQMOz{T09vp3Dg%gEeremry&O{3m6+D%042LS`&5?@I=;bJV z(>9$6B1siQkQ3om3HKJ#^g{K{))`CYnFXnlsaG zNWFSBfPZ<13>JqDTuV_kKE(!z)_Pp^7e;H!cSWoOl+Tnb*&7q4z-IS4-)o2maQ7Ybqso8O-$U>XFM{PrAeu;*!L%iGsgkQ7I;C8_UmJ*{v&s{H8G(I`Ga zUh~HAFi`VXVsb_iVXMF$w2j^Xo7m=I7m+^tLFLx)ixC>pK^r}b!@C}TiR>U=RDSzf#e-r;9b%f0-d5VC(^3^BAqfdgYp*cgY;LU(xLk+Zy zC+i%geAV1_!6g6Vel$>{$xXi##B72^N_gXlQS@sxm3Nghd-y$t<7KSHB6g7A2?^WS zYbh^{Lv#(_mUQ^{3%scZ-Do+cLEF?AD}01?l0!IJysojXo&_agc+^4URP!GLNoI$c z2n|9>#ITJ@j3t9?euX+3r!@=$Vs!ilVXtme4fWwM3idybpeF&YnC7QWqUs=N|AA@t zU_n6`ou!!>UdK%g==!H~g7H7B;o%E8u{$AQ(Vc$k%1vybn6iVNet6PUv654R)pmx!6xgV^U=wcCN3c+CWE|4&&-{im#ScTewgPq3Ss7Hp(_>TecuYy+{^ zf~!&b*UZ{~1!Y|Tw@Zb`Z|8=G@!aodCgXDlu}?MEh!Jpu|A_D-tbIX1$hr699yOB^ zgW6vX3|Bh|dOsp}*Z1Rk|ITKYeW0Wb&Z7<1oZF>^F(j1MTj1{RjVU4hO=RpM2)yGk zen!|=GLIfI%OeSMz?l#!yc9`y1WGFxQa+e^S8KPh8+io6VKz~sMe7|GP;VVQXFsTm zhlkgFir?+Kwd42Y*@F&hBuVMBU04erIFM%aT(LuYvhKO^vi*dBQ;L)wwYPd7OKX_k z#eR@Jjz={GY0~Z}czc|P!Ih>V4Ke6)p|N4*lU*IQ!kbNNc9Fm$Z+3ylMbZ0J=Sd;` zme)=(`N~U+fzP2eh;lJyCJ_h|ng)9=f4z4QIA12knJZIFIlL;{2VL{=?K^YR99>5A zmxq4xM3+~ph@NSR(0CH3GI|(+jM7X(WPUD~nE>A}v~9c^7g0_8?}yLInqu*$egqTq zKj;SBcwak!zTP*%5#>^>`2LC?+qBY~(89^Ob>k>xt(-YAG)^q*?V3v9<`H!Z5#*m( zTONX%?-o3m=rC|CR-+9A=coRfCMYh{$*Z29y7Zx0jJqL*NZ^lL299|oE3(PHqHpRi zl2U>8G4js3e!LUuFy?SK(63|tv7x$jd$Yf{GfqP?&*SO}?q1?fTu0yJ@Oz=2HsE4& zQWAYmp~AXjgpp(1Q}&h@p@h$LPF(x%HuT0~dzO6QKtHC+>L>AwQ0O)Y?FeU$k^3nKBx>LYHREuLDH5$~Vtp{lK452r5zr>XM@9ZR ze3W)xnH%S*x`g*-bVjFutVF6Lt6U&lf!~DW>TOTHu_PiA<=?Y%(TcAxyrFa$vrcHC ztmH6VGO{)YjfKnh<%fH=V>58LGxv2%B1treh*dA#!oXuPVCuogiEorby zbP^q^O$R=z4)usgf#(jYRfGZCN0~g4Zt2kZt5l>( zQBP~%7vfMhpmn02*yypK$lLMd?`t2yE-6O)f+JP_kCO=qE`%zwiN?_&zD1#HKN3o& zDexz<#F%`+=np0kF>P3{+X40D%J5<=v;L8j<0;?VDV3L>AsXNFW z$O1fOpTYPoIaDAr#l5-o-ow%I34ZY2A7J5FL64Ok zx}wV(iNN9qaaM}_v*2DCxlx5DW>m*FoGd?jEDr~lSjm<}7s$Rct}M_++eolDl8aj~ zP`{Y09t3%Tdd?)co~7PrN-13j6|0Dv&iT=)wRBGqVQAeK5Vo z3hSF1x*fLoA*)R^&p=b$JXhsE>%F#51Lr`vQ_Oyuapa{(@Q8$}CI4*hE|bJ>uj0Ku zb9f;x?QIn0ZWU}P3Plfk5*o~a71XVmO%YwJ*H!2_MFB}a{%%`UjVz%KE?Xfko&Q3f zW`1qL(a?Hsz!V=pNE+_Z28z#g50bKMK)8CeKjB{Eu8XH53q9d%h+3SvWQm$~@+$*D ze8z!4LaU&ovQx8cfZeex|Gi*LxY0&M3L5oQtxJt3EIx(fueIr(0fr}8&5nkm>7#(m z;v+-t=%Zdl)mF6Sq4&43h0l+Vsxts>Qh;AZ9fuL;OVp+BTh9kQ{L97n zPLOWT%YTjYQ^%bS`Lof=ftu9hn|=dVl4Uf!)3~s!?4MF^yWFt&R|x5UZ~V;Qq7oUp z_6|XmP@;yM-Eg0(|1m(@$_&)8WCZdlZ}K4q2k(#N5NAZIXKjzxP5~LF4^zs9u;EcG zkRy=MWDSkuCODeLo?g0w{UrVDQa#w6)8z>5{fJg-dT>dy_4j5q~fgN^A(UF1f znj0ix!@ce&qK9o!wP^K|o7G}?MC}e7xTb==uYI>(KIl7hv?@Cdl%pR2$f)Dhd*;xDOu3Bp6Nf@Nu&R5la*fJ$-xyFBL9ud3u zJp7hp*-gX$Zm{}sOIEFfekHaoY3yy8qai_kL?)GTd0~1 z`6Uf?!HAAV?X5DKl~e=Y@+33Pv=M1Q9zsarq%&3~kv&0vlgC)I9!;R3E84M5VpbS9 zWyp@WVzO`#1YFot`tz)?RzpcNSWT^nohdE(R3?j9*HcqUfm_Q#)v5V3|5|-$g!Kz(s zY@K<&R&wG=#`!#^673iuf+;zaBsJh`XRcPz_R+`ceS29X{L2{e?G$|Jv@5=ebYSd{q;328D5=^^hANR7{he8vtR$>@`Xcw^Al&0UCt%;NvCiQ(4YDd* zX(uy1ETv5Y8O`TJk~hM|p28=uLG@{K@$6OX7K$KL_Gym=RGdm4Z4F>9F1C5M`5y6T z^mm5%A|A*&qiFLUh*31U>H*bg@qIkv-NxP+Z3iEK-tnQg=@hxgZTu6^hZ1XK7%%cq z5KDT2w-f(iY&{Oj?KyfXx(jU)33dX}U;3tF&{%W+-*E(Q3g5=a_ylJ>K!2_!J*s2t z+ejdh?l~o(=kbwjV9~g^FG*r~2Aa(3-Z>@T0>89G4c0gLR*8NUW`t1B(nqm=&YYwl z67%}$t;MV}Pf_voKK{D)UaIj}G6qf$l%nBwWk~56OJGp|rsK=%Mm~2^$@;mhz9IG$ zk9sw2Jq4;o#%C-hwAJbXkB$6?`x@`y`IrJK|KLuclqqVJ13MxOwUSmnNJ#u%x)P2# zySJZC!kb+$39Cn`nT-sRvCeg2cGhlXrAf|K5iZlydZvwk(^YCPPcypTBBUCeSWqpN zgTEq%ylg77(Y~o??1jRNq#$`Vm`10mn@oS9WrXv|zg1vS@sC42R?-PT9c6+U;`UMl z@241JI}0&=6u&y|S73&VhpUQ>lgnGnCvG~P_>M(#Ap_Q&`WQZ$PFMCVh+TcuwxC;; z$J~CLc_OU1+*e>`PWRiv&CbrtFQJjtecbxoT?f&A{CGE?tfHGlaG|a-?_r!R2Zk7{ zeVF2?%i;}~7%S#9Rxd5@+`04!`kg{@nC=AKQq!e|)J%a>kCo0j?3b+CnG!Lrq94Og z;#6=0gM;l%lT2BA?r-pUnj1Gj;ELoj%n?0Nk(N z7QCmv@VD+bA2BAF=ylFg)h0|9j6gc^LzZ6id!)X~F<2A%HN8>`dym^4KtiFsTGCnV z!1l%^v0cW?g1=vGdL0hck<1e$jc|47&vFKskV89T31^;UYSlhzc%He+8r#u)Ob0B(tDZ0NE1-mcD9)zCruSWDfb ziWlC{qZy{5gR0FO=|R|F(ofp61}1e2g&fV)u;kGl<)HIUOGUP=kG@({*3+S@>uD{Y zjDoytKeQ>>8M~!myOj^Dj?NXbUz)m*1|De7b|9!XU~0Kx9?!G9#^Bj zM(G6(M5RLmuiL%J(dDz>_w(U&_}=yR*fW|%HEK6QF?zsQHV;;CE5=jEJ^XJGf(TxM zIFCI^w|Nwv#di~=cuDuOyR_?2%H5^A6LhMLqaaJ{nqZajvPorf=Nr>Z_u5zcSxxjc z0h(v+m947-wp~S9ZXZ+jz%4zAYe6QV};)wEsbss+!D?)|St54Djut z4!6B`vLI_ENY#~0*Tmj7Oh(d~F&Oe@G2MuuCb05HT>!>Lz$Q)-(M7L^<0#qFnv6pi z0W*v(;mEnuz$upK^`VvS@M}HtPKRvla=<*t?@bnZ?bNA`E`~dOsG*tZ=tln*^utx> z_UG&@O#TKC;#k6NmS)NA_hFeg&xxiZ$88ti;46)A)sL;d&Mg)qYSdLM1L0T`#~#A% z5-^2H^BV2Ml15!JCAUC1X$TYy-wmw99)AhR4H(gTJtWc;(I)g3h5FVqZivys8=fEW zzt=HD2+aAjK3u#UK6mGvi?3yQJNaHe4;j1x-Y@q; z@TX(swz$_Z#Uv;su=TEx;PcFez>so~o1sWsNww`T}5%Rj;1JYeF$Ou5KeP+w&Lm>ZI zS2_}^=t=6VjNGg-JYEov!7 zW8A5z`c0%l!`$NI+25lZH4i4465p4is1p2=Ma|=PLDfVK^~Bm1o`LDXT7~e+D6DU? z$ywLPJQ3WS8$U0OVF|(!G^Y}Blk=hKRO?;cx=hzo3V;MV9&`n)*Z58sV0`&8e7|+C z1CaRi44wYi@>QOnVa@)#aY>{`AN7-TlTCfPV>faK@ zhpkYQZ`vk$x&vqbQ*m64Ec7@L`G;hcIS%FvKdB9u(I#G3x_psGt$dqJ-43QOZFYQ@ zp^Ng?yrElPg{5Wl)a?;p9 zz41OAO<&;ENoZ5a&pSK?@S4$FbNZKHDJI2&jSOO$jW z$r|-_pY7gmmv78M#-x>-=~OIE4_(4Vm!NAK2T%$qpA**38*-x(%ebUZBYK&~b06}F zVlT2v{`!|&)9*eZt7(8HIke6lH^$5E9z+V0?_g|w7{xpgCck@#tFNQDgfF9@b;7E( zPd|5A1hM3`+MH|TQM70)o(A>0ng%dum2&Oaymz~_jw|WF5Qka9!`}4!3J%Jp{DLU=8CC;z46UDs7!FP@l(iIC^HoU8 zrs<}P1`U4}Db}sf$bk+C_Ws_a+Sk6h9^L}5PIvR^-?Fqx&t+Ef4%glL-gIOE904`H zH%_1Ud-WnyW(qYp)a$!bc?$Y05e5_^>9afGI!sqwb!PO#M4p*9P!WbiQ$N<6^9~$h z4rq!*oF)mx*PgS;a_dEqT*4NeT$#l41_UFcKD%-8&n1hY9#9D4A_eq$kGMY9BjnIU z(wR*oaD-M-?w0pqkVH9L-^WW%`g{?+hVeg$ulsx#(G$k2o?~7UbjddzauDIbh6DVp zOqY*;q}89toFvtsqbuEBv&7K--UK)_jp&-=6*9}pZfDsIPv1RU(@@~>L1SJT+;e}l z@9W+Zz-c_q!aqT0vrYqEMrYTr<=)L(rf?#r0iXJS7oW@bC*3Z$|8Fpi!vD4yhItSY zHO?8WAG83<5R^sG96hH-w;m;00ZP~xP`G{X>C+87xM(#ngI}qy!BpPIhsKp=|94%9 zf(Av66b8CQZ&dzkU8%}X(-Y1|J%iC9BTANB-@lcf>6KG zGL}#Bq!D;@8ddC!rxgtUO3SzqWxv$rabdD0bi;B*-}e$|+ar)}kEU_Y>yO@*n0Of4 zTali6v3yUj_|%zbhiTpG!^0&qhT67oY8r_#W_({5061@N^F8}UJ1Wd{An=Jv>>u;o z-O|8?DVmR@dFjZ5s7%rz7Xe>Gm?PCx?EFN%UYwkpQSx9Z3Z$TvidDa1`ep92Og6r=tFHezr9xP}a6^Jy7h^sMG6=Z}l z?c7h4U7LITe(ej0xrbZb^v*`Js%by>5|X7J%dg;}KG<)?gP0%RA#Bh63d_tBfQ${a z6Q#k3G(u{V*78Ee=PX&{nJP;VyU0+KTvnr8p2ve@HG4v}Cc zEQon{iYi6G2+z#yOyTxIlS<$)T#r+B(ud(@3}Wj~3AP?Q?jZ(}Dei!CwU}HZrjGcG ztE(ruFBvYEsa@xPV}Pb@-VV&)B*qlTA&&fgafDpj5!;^T_VEPH?ne+TMsuA(8iV9U zvqn1P9q3X`XmSKf<&I_AdRn(ug zWY4M}V(q`3cc1Sl?fPs&y^|)8j|KOaw8!dsn6cJ zNTE)~MLEcyxXSn^QH?~WH^GL2bH5P5KW{=lx!_at2}MYoPtWF1o2R=&QIZN^$;G zBDBk_cLoHrn>q=>)f~-T@vbUs&P-7^jj|dnhqiS!ikggfP@e`}T6#;W-*JjDG}7B{?i2WvpXPI6v*WKZ$1^lmAU5_&AALXPE1nIyiM^XiT@hQ>5_Vad!Fm zQk=+{)5!@^YZmkE_}%m4km~dE*j9u=O{cG}%#Z?YLrp^TBV2Aj&1415 zE8g3?25)Y_GLXraMuz3}{qy3SGuHVU%is9ID-ays-XD(+Csjg<7dPBuW9@7*rCQqu zLxw?uIsBk#MH>%!rJ6>mJ!3gFQ%0acpCSsuog(B;@880)(Da@0mtr23ymCBBxGFhR zI71YPsJ~i<%lFadYU18|WoXK(6*>mVR_K;<^eNqHWjdf2HgL>mfnD`PQPmzNPSx-t zb{EemWYVYv87|mZu}N5B+o`!%gM%=vbvXX`-&o93>{;ipfwARH0}*wv6d8yxiw|?( z2T;i?G*Qs)}t zH8yXdY=eMs4>6i>&hkj<>&_sU@cmv^gGS)P{jlK)KFDoI`s`98z!dmUc-ms35HoW^ zywYKxkUoC=O^#=%@5yV|udY;ePT@f@%Fu+XbK&_r_# z7KJ}hA#f;RQePm1Jh!7J_KKz}3RN~V+_0_1tytGJL9frhoYaVJekS)-!y~ppST~#t zSO!eUuS9KBZAUgTQfprCK<|>MU$V)Z(TpT;iX|d7_H4hkAr(B*F@4(j54X4h<-<*2 z=18Keh5F2QF-Il0$HrgK80eAElYv>wL_w0Y(`2k_j*%AQ@#=I1I@OFAO}X1>t9qO7sGKOjXtdV zdUg$JNR7cYRv~8kimX`uerbJWWl|GomM@Ir=@SqKF3Ud>XuzJ+jO^DyNJDb}gejW| zy_Ad}nS^*ApmNGFBFT!3imq8HNgdGtF^SqEW-)zE-A#t&`_vZFskccr7h&r8wq?DN z>q0mCO)oalj7Lf^d04NI5l+n1T%CD?n2!G>b1qPaXq~P2q`|tS&?>E|aJq;XqDro2 z8afmDB%pX=O#{n5$sjJS!U{XcT?}a=P?MMy-L<6&1SQb4++3lZS-vpNm4l$nZ}2WH>W8Loi4 z1cd@V&Xf`b%;a!ODAjOmnH*%C8R<|jBxi?bmhvl@*EEpSPJ_uU2!D#706}P&1vH>q z8|{eb;zHuNw);*4UJE6p zTfm*&?}pUN-y3;QpZ^}gB^CmF{H00`=h(lNX|ECXXp4Wsl0*GlGYIZ80tXNw0x1us zLmWU1(#Q9eCKQ0au}_iN=LooB6uzL5yldI1zxa@21{hu+&c9`cyzrMHrd>`ElH`C# z%}b%Oq7`mJ)hq+s6Q+eWLItjCmk;2rZ+8%S`vQ`-?>!DI=bo~!BYbVN+KasS397Sh z%Sj||>vuwSnsG{iNIkiTKfXm<&(w#k?p)v&hOPB);6@cAGzOBgjF0ho`fU5|9{zfU z|FiDe$Mk99XQS+PgP@AXt=#c0mCX&=a!9(1fzG+3k?X!L@H|O;j&dIfccUrzR-E5l za(w;PY5dYnb-dl;v0Pj`c--wf!&lM`IDdlA`by>e6)<7DX#J?VeBWLV@!fWO{)>Pi z4aEAlfcZJkEja8wj|71j{|eW7&3^b0tEX@ZBY1*U7<8K{EJ9=<65`;W9MY$3F!ns$ zY;HH`0GwHGoKT5Av~T!#TnVQ($)Vo2n?8{d7_uuyb>m+Ii+W z-rBVF*y;KKI&3a%YQSD~P%A=H-)zFIv`SovrBPH)2F8&xFw;6=z(G)@HNE!p{M?GC zCgeIpyGDFaKWF0ICXWJVA)EC%Qab3CpwTX`uX!bb zEz{xI%mLAE(TVc9fn_f(rf1eekryfRLT6==-sNd{-Tu&GLnbLBeo>Ou&bIsmNgr-s zHRj&3O43O)nz^#CeWmp3 zy0IBOFmBV19Z&JcABcbVm6T(R(1zt{!KdRALz|L^cq8ZEgO3QsVG4WO41oA}kT;zh zsJ>IT2k3%L4np@?4CIZTIgM0;{_y`^Lwnes9vX_s@T|FtW^{JqDXGn*bznV470vO8 zfsXY{UyF}VE#M*vui#B_f`1Ojw2$#zzkadz8ofeN=+C(HyA7ub_&Jyh zkp7rM7E39Irdp%~?j*PhZH%&Zq4K!zbF};cap6QX5jYqJ*2?sL7LPXIhkN-3 zCrgGvF#dCyg9gn9IyYAYlgGxrDlO7uevG(Z6HCUACK$n(EfD)-{OInTq3&SjAo05_ zhVNg)Vd3fSspXGa#yfIZ%vpsP8bmqq@S-hQNK+B%O2Ia#-Q(<(pcQ=lVDb5(&oIM; zxtj$#a4+@yX5R9f*-tUTNF^n(2EDykrEEDLytkA2?xr$ESyl)ofatQ zW}n~Z$0#G2XsVOG9g(bA;?^Zn%hU_r7#hv{U;I%LWA z%*V`*>8U5!J^AXX`E1W>M$A#>6(Fn=D^J%R2DPAE6EjzAMY7Iej{kT6l7BnjvB7#8 zTqT?e6-~n0XHbO1+2hSGflyp8j|iwP6Y%!#4k-Pc5AbSI&cgn7<|GcD z)9Ig82&v3E8@p&)+FpwEL-)4Ivlv(z&*Of33hhNJG4K?Q96dE%m8Piuyu`LKNWsn$ zE4XkL<^3yb`HKJrMeYyItkJ;UgQ$-+MC=#~uRT@6Eq@xXJA?i%jp|isBD15gX)|>X z3*v^V^jrYeU5R#vK9k@g=s1A?52|j>Z7TSnU(5A843yuqrUog*0MCoDs9|$>B^txn zTE-Awe)BE+ZxcaY*Y2yg4|JXBN);Ujs^AyjFzCZ2fseAqqR1EE8fE>Eps0)X zs(JV{DNXr)X}DYI_DbqU%1ET|ZGYx=qR5moZWB^H$L^%; z5VsB#dZl~DZ;f|`@{s2^bF?!!>_Df<9IPJ`x$VFfQU&|FDKqr@jVq}FbGcX@4KXUM zQ4~YorP24L6I|;Y;>;-WdumwXIbdGiSDmUg?_Dt-xso)0#i@%=lz$!0W7=5PyNpQ^ z;44c||Az4s(d()y0G7B<*ASHR+bf8xEbv<05j-0V<`81FlAS+Yp$iZdvod7xY3{6C z<7>PZo~GK1W6o>U@Rs&dMX@Kq z?Im^xA^f^hta9*@7;APd1v@9W_N_>EqUQ46?gT&bPgl+H2j`Z+x;C3OIQRN=hNvoO>fzeQF)HO?3-Y5LGJ&H6 zl#h1chImTRjBgTZBiV?|X!&%hIm7RQ5Wv0mk`Lq+W$S~B6CoMX{Qx0{VYY-d{~&>0`A$6%l--?MBd9~Z%j${qZkA0 zrnKBW$tkp8A&8p%baggjVSEdop+Rmpe5vHHezSqTvt6AoY%eUZf6JSUTVAb|?0&n@ zFeZ4b+V!qkgS5QR24{3#=W;wdx+>k%wn|5J9A`kQcpk83aU6{v5An zX9c}lqL*V$>{^JVcQq0a?~NvSoolM+N{QDmMY|$rEhK@iuW?Kz*$2!BJ;AA@ zIuncLw1ePJ|67gi1<`*A`(&=FVt5VGQ7WDSu{fgx;e&8@POo;nUUhDXm?h#fzPLUT zeq`91CadAYk#$bU3%^W$S$fILUv!xU{}vc*-e)7rUiB2J3f}9wK6tU74rYc#+hDgf zpcPCts~`;8?&bFVw|a5;T}9i_9&cMpg`KaBBi;&AD~u+ES`(RSwC?30#H_4BcniQTxV`{ba4Mrleb5L z#mgoDjgjX`o zEk4J%apWG2322=+T^En!(fT6|WYqSx`)27kxW>&SN4isO&jdn^pZmsrw&l*VBD!(aUIzu2e=zy>#|#0@g3^88D)%C%&wooBrR!r0n_tn zJpX=|AySXN_Q#2=_quWZ7$Zho~c-{$<(dC9l#_EL*NPXkkRJHe^!tk_X{d#4z+s+n(;o7jUfsU3DCQX=KX|w`0OqoZ`ZrikJ%^k> zzEH6!#;B!3TXep6Hy^LNueuMrq@ctakYkD)f(SENEKo*^48JzhuOY7o8coKaYzp;2k`_Vr0!G8;5E6J}@s_bLM;hK_j5W81A^G>+nO+R$ zY$G(kv3N!sS%GU@FSQ|ihUb(n79_b$cjOW)L-D?gC_c~*XG>;opF4O4R`~axJ8pVZB!J0Ym zNanSXTQOCqz%_7=YvQ);=gSz#nPSz0Xyk{J#afypHp;ZFr=p?ehs|<*x1Q%yMGKm( zHb$Ib+fh)?v6`q}*>1*KTS_YD>4--UTYiLK9bT%#TuR!Y))`y`R-+Se^khywv)}Ng zCJ=6AH9o>uR6$pDhU4}A1m|^q0ubqH<9)QVd_KKIpwpjyZuftn%YAN#Ja0?_EVuMi z{a_njM5<-+iQb!UL%q}BYhPtQWANWC-3I~c0@c{t5%hR>YkqFXh}CE;@83amXyvUr zHSI=SKC}GE@$s>#LuAu+w@6(Szgm;sDCey2N?5jx?U2aUN1`Z01l4{Va0nh1IyM%eO)y$1ynIudU}Tb zjflA;x)T_ZmF z7N+KRa%5cPdvV34gwLDxi<@Y?SOm`NlM@)xVGd0^2o2r4JQ-2K#FW(z>!2DkE4oat z3N}F{*$CQz_4{fl6E4p@tK>J^(DQiBE+zr);>;EW2bukmk#AJ?>xAcjZWaTX79)b@ zoD;yO2`pX`A8_mQ@3QdvkxhPTwL3@u&=vTZVY11{(T`1-+-_7X4)|Rx@n*pZ@n@pJ z+k@7200WgH51d#ke%nek^7$wCyag(WGVqqlK`5%=<9;Z5g(9Ux(DQrJAS|+9QD$x# z35~cz+3h^zh_UhR{^|WmcH<$RUQ~l#R>k!dxO>Y0bc^uk`O|moYhQR>?n7UX*y-es z0y>yFX-RlO_qD z&9aH(Jg999VXD<63#_k@%AL!u=(w4*QIjCYscf7MioB~fS@#wZpcq1}Fswy5>Q*_L zZC#g1gxPrf--)B}v&m2{JY+W#AoSWA-f)pv&&CuaAfq5iM_JESa z)L3B+Us*x(gvDT=W%;rUe93F2thV-UPtkty(NRx?~co zAS-X$CK|H(K0b=6e0w>@{aRN#aiPyQV8Z_OS~T{pTojk)Yh9^#l%_nnAXssst&Ax+ zeujkQTq%Mrs%_j{(5#S?UFW)MKQm8!Y;^dw4-JGmtZM}GTt~J;nxUg=&ujBl9+zSH zFcPf-3cG?WC(H+qNnhbRu*Z5PNgY;;I>~EcE}c1}`A#pn*g?KXWj!5eY7ipz8zU27A6dYC|9Y(Zqq4O!f*9+68S4eONiZ>yj0Yp$92K}C{g z{$r*{3>O85WG#|;{$*G@dDrYw`wtN1`SrpzTD*4LcS@YZQ1wUDz_BG|OSF*IASxO` zu|EtPkaMQoIH}!Gn7}!?Csy@83w65^u**#*&?a;%1V9gT-y8LdcvGF`AweG9M~6iD;O2XLsd%73kgl{ z+y$VNS?i@!Av&-NBzo%nT~#{Nen<()ee6mMmIp}Aht>ujSf%UKBS;A64EXJzTr9X| z63dpz=bPBqin+KJ#>q7O&5}3>V=w6x?;+ z4fx=j`eEeya!c%ukM1TyT(#Mum!)&sQz%lSWgHc^=LHpaq@`zY+UP0Wc#;MMzy zH0cx@IIjs+4}e1rCAiS;`e5Tmkn#l)J_3jK?;faMU{D7j1Mx2TlnHimtD~mVMNTwL zX9Q;?euv=(-7U4iqvu=1+K7!p5|d$wl|*q@auEV$Ks$Wb5Xf(-V_mb#9J25`e&kXP~zAU7zn19p?A_)LW z#&Ew)`u30v`_q*|;Ul1evjJhz_At3J*ut}731J8^L^pF0?u2+#Av3y7>FZ(9lQq`F zj((h9i&XtccCglBk6^}Uo%~P61G~mfMQzqcRA|#yBa1x)Dra;Dl7)QvYAt`i{*iPU z5y=4vWAl-5=E@*=%wAyg&>ZwJfVh@Q{6Gs-fi?n`<{zP8zc5^mKF8nD~+3 z4OtAr1^iRLs6ZO6VB#;r#vuEc(P$F1AIY}N4N)uXhScnl~V z>opFN?XGCBX(h^n7K8R4ibJZMDqVKpPQ_NBgLM?6oo_*g3RoJz^QY>R>3b`etg?wL zvTTe*!SJ4&XEPH){yAO&ho89X_V)sxDeDbYb+Gh zBdeUx2CMwLFm34uN?t>uLe$00R9Z|3Jvy&Ieix2h$e$_AO%45Muth{C;g%2@9dJa; znYFAES08mdv-!rOCwVN`z;FSR7->2Po9LzNdetq)M&k$OXF;kGmZS%|u$Wl5TXcej zaey1oxa@xR6B=OS&j8>k>C^OkzMXNrM*X@>PTE=Tg&0McODWn<7vT}ikGRNvA;BO$ z_(?K?24x8+sjW`sO6pdO6$`S5P6P^7BX`Dv_9SB&t&~BjqF<>AGpc_((~g(~wo5l;EYqqP zb|h!%DY3u03AXhh$;Z!~x^#XeAT6oVut!#=F@3Va*fZD5u?5IPIPxKK^pfOYD;!cm z6MF3Rg7nN;1n)yT$V6KqlduKrgW~L{q}b;5GPH5sD;cceF*BLFhQV4)@6V2K?vri$MW~}d_-@VmA#1I|_{~VUw8CY<%ZD1IYBrdgMwB{`Nkra}dL)yr9 z2(B;JOd|exp=o_m#7Y*w$CP#v=9P>q`@ptl{j__v5 zLHMMzmGJ!hY@$v|p`aaq#ba4!eB%UJ%ENqv9MruS& zJ^AkN=h}X`8N0$3i~GV&v^dx`)*FbprzApaC!Qd&&xNwmh9**-{YuxFG(0e zGpJlq7PoPKbx5UUmY5m)?S?>szycqsD=t|M9ujrsSAT>yQFDE6a)r21&Dq?{oWop( zq*SKQTWc)sT5X_Xt5u`(pxbAR@c|4tv5Ujq-Q9@K-Sg#-kLBtoPXUIa#JOXh*E0fk zFAkIH*G-42N>}gepMV~Ftl2b-Xli#W#PN$t%xUF6;-#X=r)5wAQ>2VeboJ`rL(iA$ zza~34wcxA0*|nyYh|LM6mUx^H{ND5*5mN62oDoxRy)_wl?V3W#molc~!O&sjF1`vd z`gZQ-3;CJTMT)3Dnv`n%&%3(6a6T<@8Kmko%zbOhG}38=q$4CI(=b(jZGgm>hGOd} z%P#E5aJGJxgXOmYRHPxFZRaLsN_zFYiO33nnr{W$>wGIId?Rnljl}WKkJ(Z8%AYe2 zt*q-bOnqm_YoQ>)hHzdkNd%@ZW2KON(fL~Nq2++eVUYc}Ue(DJ6O5moHB{Duvfz$y z^x55PFB-&ID57yWw?zu;6O2XNm$IxasLVu?iR0RHTG+4-l>;q(E9iPXSTwB|Y{*y8 z$e-?2Eu%V^)R4B*X0pW_fC0h{>FUWH3J*$3H`&+`Y7P>{w0uli%T0!XEn`_R?e5FK z3k>bcVIaLt@s>(F^Uc`xCa16n5F7nEdVD7>nb@aQdlvHA*=of5GMZb~?PORV8Fz(G zMNMtPn`~GEyz0=gM_qef$5-!zO6s_@J>GVbE8VgGaW-Jv)7-9$&hN8%Mzs7;z z$w%hm7%hdFDa4{h=Ai;bvn`00F|ZX4Wn*eJN97evEEvGknDozhiXSoqXGb6CNv8&Npx1<#VwgzJI!!TCIsCp6_?nCIqG=7oZ3z*Xt(3?-ONR*t#5X zk`69(#txYT^u!IoK_DR#C!)N*W><8#=-n;aj^TRz=#gki+xJ2Sh>i=3zy7w1-j+E4 zn@(j8OHda;eTjqI8k&v`RhS81hni)YxM^nUG?Si;gYUrib4uci!Q@!CgTn;+JXtYUr2Kz8CeIbkGq;*UEwm4wGS5hVx73QZMDjhN|Ly~+@!?vGizr5L;Eto};NVadY*E*#3`^tioUM2bgp+=;kP2sZ4B} zeuCa0f!CJ5QaKM6AN4~O)DI`pzDP%sT*PF@v>D*50KY&9wwQz79h#0DwVTN1oD_y< zb2RsB=Id`hVUYbbu}h-&NP_hu5%V;CLGQ6YXQec8yn<$aC|e1v!sLw;z;UPz9&ajl zME4JnH-#39mgr>G#XH)FLw=n-Ni9 z-c(T?5*0-C4(8g4|ijtrJcW3G5e9RNJTgGjRik8Yu^v4 zS_lHb!$YWU_r0>I(?3O-s?i~~ z9j!JdMXvRDYb$M-b0Tjq$iJe7?FMj$Gm~A!GHTxY&ZIkx+R+DiJ2Wzg8)!=1PNz~Vp3{Q>xzyzU+{PyET~5M65i zV$k*${jREk!O{FZPaIpO7k}&;A5u`@_Mm51MwDf=YGBy}s8oDE!AKh$zVI|5Oo_6hiXzw=7xy2k>0syEzv0a5_{>jV~nY_JzSWaMN%3E*)v&pcs6&t)>mBei1iip;8tg zRm14Nu*YS^&;1Onk9y#^Ge#O0mF{{FQ)bAAN;}=A zItZqJlprrJ)$}2F&2^aNY|hQM51B-q)wuqHX`KJTG=~2j)3pB=Op~{B0%zZf_7~GQ z|HU+(F#pCh%k1tfrYF6u#(3@K%k!v2QvaS;`Z5wcT$lOkYw11B>&aST+H2$Lg-OOu z`mU(?k72)@8sf4Pau`F$a5@HB$X5lw{nL~2U%I<*%*&_HwNvOJuOj;1psO}bEr$lvCAEKUH&{QE;p5Ij8Uz!yB%!HY3j9>iK;yCfYW3H<4LdWK-W!%1aw?!SKkeGbBl|}_UFpn(!Y2ou{8xpJp%Qdf9Mc9e< z5N`C+p20I)@JcA~6}I6}*g@qVL;iZU#sZL&nSII0gadOj_Axf~fSimuE-)uU{U;~mOK%O#$>7)-*-q6$ z#&48v19CF`mo^4}axxbLC90t5&x%tMHlY*Oa)#HUPi-0;u;)aV3I|}IF3+x%gc)K+DQfH7r-^FrB4*FzQS9yZSt|+*wDbpB-DWI+oQ%4O=nxU?pzh0< zX7;~h8p=Uv2k8lpt^5?8wU;qX!|Rwvkl?wYf149G8nH=@$L!jGb;8e^a6oQWYYH%? zNvy`I0FG(qzQ5P_Gp6wYj%mQnsyjO!z#L#ocz!Biub!RHYqrX$ZmT;*^my7*Ry*?7 zzVJ#pKwjzYl~>Y?-e~JHL(Bi_o;ygFY2r9+l2Bc1_{u9W4O_lNYy$twEA{@1SE>Q< zO4c=kDWFcj-faPSr2`>GR4fvGvZQ(C)PS;gvk*p8|orQpz7*DeNz=G(_E9 zw^D5olrS3-K_l`b1U4TZ^WJQRpVg74c?>axJ25CeH7yAjFupY%Xp=?uUt z8QXCY@m|K6V0aKjXXncj@;gCezi&3EWziD!+s$WDPc_2kG3tnBXgN;mQitD;(e_xY zu6aarUyISEsT4gPWH8jQc!FrY$Qegmr@bBKqRP*gYsC93>rT2NkArk3ahdTm_nulm&dTjLA3>Ue1C=4Rm$nxy{3`vEprb5gb|iec zk}igI6BnWa7jrnKD7sk3vX+n8L~D6Pm5`3hn5eUNT{Hp8kMnvbpqRfM&%RCzI)@xBl z|8dw^IM>#~^#TrECTyYxL9MIEXxg~wE7y&KXp8Vuy=AG?5fO$5y_Jo#eTwY>20>G> z!!gRjB384qF^+>iV}pxUwu`IZM@)9eiac4s71f75qr?Qg#0v0Dj~uU!bg#DqO2v=# zyZR#R$G!_q`r3)gEnV#!e~U8G3@=5Q)YqcSahRp?Yf&cYwJ4(kxs`frI#Q3WDlo@E zTL8vJ#3o7;>P2sYt1mZDmrTkK2Hi^}?;x;O#W64u>qRdOEXrsn*aM3)#s4kJyn8Lm zcyB@lusZ9UX)lk5t#<;7GOu$Q$Uf=WQTB!8QO{qG*hTpmrFSj4?k!frDm3=2-NB4Q zm|s5BD#K(YPwLiC$y>BZ=AA&NB*Rm&Jk_&N`2+~5n{3)3{N!y0!46((H zIiY-lktVAyDyvimdZL$uA`>B_O1mGpcuPoBE*I5% zA{$SeRhVKNmPE&`v`a2yF^U06ne#tM8N^hIBF#>UGq-tx;;NF%Om5C)iYui&CfaD$ zB2&MzxIP2XS zZ=>DtCggWL%K;>1Hh%yfPPT-N{`@bmbhUp$|H3P&{*za_2JlLkEjRl>Ua9UMyi#p8 zgz77=^i)yt8OSS1>r*O^rHx6e0C*+x61f!ES`?Kz=0&ScSX|Ik{t(M%h+7*Yk#^;? zl`4~{gHvx=!p*j(+r_X4$}XU;^vhsg6QC<~{Lz&Z=sY)ox>DVLbS2u4T2e2%Qo?_A zCFfUN34Q`WXoc;cx)K{uR|*B{O502p4TBd>O+a1AiU6o9i8^-Byy!~t8Q;+r%2B7{ zasQ<&#Q=4sWlRwjW4#3U{(ICHT`Aulpeqdza?ZZ!N}tctbp^8=Qi$p$pAPto+n!yn zZsx0?Zjb5t>fE2@r9aGiKu(A&nxI3m9^LAxjKPz5G&%8+qDq!^SgYh1JiuD!lN%V) zK99c?Wl9PD6lD@30HgI?66*H#tZOdhZ@PbVB~BaIvcW&P(!r=`&facJ5kOaxc+r*Y z)?K@Taq|-eQE+bsAOJ;~NrKi?xg20o#=LU#r6|*AaAmkWi3%vnm;;M4XSlCL8Dp1$ z9zanhe%|{Boz>r>Oz9Rm;cHPwbvfHLr zkK_tu-dsdcyE{pGgT_q85Lvu;#pa+=#R6G2aLV-On}_HxywYAbWqvg3_h3|U87KLL zbVJ~xCTpWa?$4qo-l8dg$`LW0`(`ond5131x&7y@_i+JWQ8P;L!0&u^i}T=(+V&Lr zx~K{If?T(Meay!5l;njWiU(yUw0NyZL-t7-lx~p9Zy>LOd?wHa{zIhf3KeaUes!hr z<=$;!flJAt5-dgvrn$!&9InKE>dxQG9smPY9Wd;mDDJK77t zCW9M!NDu>xGJN8sCP6ulzSEA^_TpK>q=2H#EO1g&h2~}-EiRMENO+0=Jbjyg&P-cS z^+6^BFsT_D;0#&!<$NNeZ#ww|zH%z&&Cq!A>9)slV(U4^yK<!?h2S=|QSP;$nZ z7aW$#dEzJUXs2lY-yNs{f_2;>Lwg?J`zdsUbhu7G4q3CwNza=#VsRwpi;D)aBvEo= z)YFse%%-$Q6i}3k{5CFsbC2MniDBQ$eVS9yg&0qS$Xyzk8sh9akQ|vUMVtn4&=&NI zWz+MU4iqfX@%w0|CGcgLp0uz4as)HP%d9|3Lk8F7WPs^bSN@=6NZ_a;rFc6HkOM2c zmF*wE4#9>&NswsDFOw>xi#FCrq9>EsAZ?b&5#-ov%94tBuS}_l-gbq*V^UTaOC+-O z9)bF^f<3AhxgB;t&T8{9hg9q#Y;pEvT$LspU{WK+MoC=*oYa7-m!_aDpcPfx6Wa*) z0C^=#ol1@@6dFA zk8ck~b9SlMkZWgtR5vn$J>Mc(hTza9anB8C#drMd#)L1g?Z>qT(G#zBk%*;kBCD!C zc+K>jyhnXb^n5iR`j z(vo9S-Y%^N>Pq{<1ks$?5E}ivJE?%{=+ZJ_b}GZsP_A-{5(08NBJI9GrA7Nl!+xnk z9a`+0G#110;(@#UaBc(SU51%B2|cNAJ$Ig;cD%nI_dh)~mE`z6dhT$avmp z=@h)`O2uDdzEseK`!JuDiDPOZPgn3Id=F7%eoN2Eb&EfQEhGd(j}$~mr&)eHl}Yeb zgiUo$)9_q~dYX+UUW4gN*EKGh$tJ>wGKs;=K*K1Vi8zBeSw^@C*Jv_06bcschV~4M z`87whM73V-Oa-53A4H0=V1H(sO1NYpeFAgi{pgcQR}PWQ$Q=l!@c^ciLGWd8McZ10 z#LKid9O#=IJO%I8zKk(naP#%Wb;cv5V|^Qo{b5g_Lj{P+d={|x=NwjmobJz8vcvTP zMrF`#*+X8UGBJ#yD|N03_Q0r2c)}2T=?GLXy4{;t;jzC_nfgs(euUTxzDEUMR0c9r z>=uz65S5|bsM1F+0z_q2fl-;hn0i+0t{zZM@j6l?f8?!jQU%epneVG#esXY%Rl~36 zRI(0Q6klsuMcQ*0cb?u++e((dL}hkdD+|~-82ngZLxa~??MRG)n;PjbSt1oCx)5k? zGTd`_WXag^*s70q+0*1&6>r}{50=}%uUO!Cql>1yvgR#e6W|~%vjx7H+?+*6r|)7b zQ#Nog+Ce8v)$qA2q>o?t;dN6Z3E0$F05&y6mo!E*$N|7j%|x&k9B@;^2iVjk05>&C zVLnt<28Yze;?1d)+7P?pm@k_e`Pe(n7^%aijyk}m#!MpcgX7K3d>Ao(q5QcPXs*{A zsqp%n3y>BHKvV`+>6xtP?K_c6+6tqCKT(;RwH*^DI6zb;&M^Bo-Rfc1Jq_H>$&cU< ziwI|P=_1_I8Nf}A5^z(~&>{rb)Wp9TsWdi!(4T088iyFoP-*jMmR7Cz26sB^c}Z?!E7i)WBjg z)tV6Y1vMBzR0a_kl@aJkjG8to?k92OQgRGg^cblZ>>WDe1i#0US~A+K5&8)E5|z0t z10aC`*_@2NzIn(HQROH0K90N;T?u1F#8lm+#AFI{Hbz@E)5{+Zf?7Cldr4fNx4u7Do#ERYO;&GAH`ZfkGrd*^4s4LA2 z!En#3Eg{jo=t{pU7(R8s|BtRTgbDZ|wDPa6Bn;4%rkYI)GBN%;skwz6r8bE+Oa(`6 zeVNp7DRSh``erE${?MTahj*b5TGs)$b11OH_W&&vR?a9Oquwml$$y#D*a0Us;47`+ zoTFcIH@stoz!SVV47XVu#w^n>;>QlOUS6zP|=CaQe5iK3a+HGV(Ke-cKd%| zArVxr&qH==v3{M@>?(GS^ww6}Fl~_W{3PkP_|LFhQBq z5*=Q2rN_UzlF!Eya9pDDJf5&xk=b@4o9OU{5k(t?8)sbq+bBWw8PM+Qpft_$46RKi zzrVUt1VC59ilEO5S_kM#NY2QV09^@z%Fe8Cv1YX;1CC@8petFx-RW1+MlfuP7B64i zl+tf$lCw-I7-zg83D;*yvqXfW;epu-$aOS1wJNa_0xX(tZ#6F)Q~VWibV`UEVglbz z%N_}$ptN8(cM@#@Cp8ejNe$aSCN;MiOqg6%Jb7A0vKXEE7z(|<`oxtxf+$jDLHV0C z;;xiI09~n?5uht!zv@ci2S8nE?2oQgNq3qH3)Ge3;C@ehABVv(39nqkSo#K$RC130~MY%w=k19T-=9N}Hspbtcz zxFk|JDAfoxj~KzLN58N@_~-T6RzfQ@D!?L3+GMCXwBT#WN{K9fz0sJ9D}`jVrM*{O`@<&%v zkA_4w{I!kty%wP^?ww-o#jT}q$0f{_`r87Pmy#C6aZo=LQqsluSF^vjs`(o&OBLU6|#r6+7qD(-cE6NQj;N9FGMNrgPyEH znQ&}(m72e{k}#KFH_Lq$g4MQt$M|g<;Ls9fTaCOJdWu*Y7jYUMPRoe*BPHM)B;@MX z?4+Rl*v$ej9mWFi2>w6_=^OQ*{pCeRO22@*5;lL%bOo!J7{AEpT;QZ8c1MPZ(Cn+* zvimY%Qd9MhNsSwkVFT+|2di$@T6+cp3#Q8K7=u-oVp%1cYp{qDmv6$hdJ>6pm*c6E z*qBOPbVf$VUw};+rpll{O&L-SMPoah3VJ_aQ)bOi8@x)m1;vE#mX9_X%b!o*?=S=B zB9@HmzN=KYaC3RBnTmsxm6XmIyP=yn#!3Nl9JDiRc%OC}1+v1eFRl!H6Cg3~;}kG?`{}mM6+FHQe!~DMOX$x zKq>eZ@>2!fu*e)I(uslhmfg@fnJAdJig5J_L=4Bs(>(`ayWln`E~Z6{4I9L)O@pbE zry%Am)m!hNGh662sT=Tn8ksCcDw4SlB0957#1)8HHW zpGm>n_&pN3pD{5nEA&Ko z=)5b;^*1V`{1TO+P3N(#@A(sz$#R`VK1l&AYA`wRS6loc<;0HUNBJ+@+cRf*pL>^@ zm?GZsKR@O_=0{w62gwpFKQ93uaEgSiy?trQ?7uW+IG;{S*+)XveGXmKdC zk{aPdvGA{@rLcs>i<~!GH+BK4Qq~k^{TS<@FOgpp&fuv%{_-Rym<%ur_ z9*0REg(gEpU!NLG}`Mzzrg3-@caB+sTvN`6Rw$=SwYa zoBQQoPl|t{GOHCvB$ajBbcyq%0XT(|%Ba4nQo6ez4J+`FW=lLF58d0kRIH9Z!|{8lDU`>MUw6dYE{s6UOh zC8Y@j;Kcu9=UPKF4!I-^6CJKRDZdjl9vz9mCnY%?};3>vN~JUZ?jbaclbw1BkUF zbvf|&+ur&})_F6=wrDkR@H~{_oAG$DNyFpt`8C#9`ZfySqai|Z>8DTZ3I%sI6sTn2)I&pzs?c*b#8Yy*fl=kCfP;`yv?{|s?C(PloS;jc>Vwt z^>|Nlt^{t_?YlI@Em^X;Q&q=`TC%7y5iY8{*cmLi6;5p;D6Y9KQ{#$*0{L5fXDc{X z9YZhLjK{irffQ^~MfntV#v?xWgCFvVZ)Ge#(e$UD@KDN6b`le=u>v({JYqs53+mYS zj3c4N8NmV?V~_`-AP+yOkIS-@?bOEcA4p8;6OdF$F{D-Ug{<-Gk(IubCcL0Z?W-0> zn$d;Xzp1?j4x?4EOF6bT4YW+b1FbF}wBBAlTXwFWjZdN6UJ@-PdO*gabD&15Bc_zT z^=i_cCdz6XxjoKPjIWZ>g+j`s+a8!m&UX5lG&Sz4IKAJmWU_RC-%(*{Lryd>ygyKf z?L1P?23;Y-C=s9+lsZnNug@gVtB?^y6v8q68*>_Bs{WJ*L5jE=JI0G7wjF0Lv9CDm z(O-_IKq1}&pw_Lh>+l8QCRLYN_1 zCcV8b5LJ3pG~I+AcODg<`70cR4o1=;A#BOly`#ymL~Yl%*!J7*7bf&D^x(n$pN#PD zZ4r9WedyQS*3NAwz{voA#W%RvVQB2d3Bnm!$x|^o^P!g% zybu01Wk^GAX#X~4!ZxTp<(AXyqJ-B0O&LziKTVmfP0iP)Ogz}vZlzDG|5sCn@3kod zy~^s%$|78{z}V=ptVOE`ewi@rD?rG{B^*O7R`tzdwvc7&>8} zb(BV|g0gkeeX=!fU#3gbw6V*Y93e8Y+$>!7y2Q+0?ckiPBO&NKOuqo>G>djwW z$}deBq+^4Zri=piP#1~8M;qTMgcNXE$hC8fRE?wY8}&?Q>H17sWaCs)n1~;#L;i1V z3Bk!eR=ScQLYk~WwD_7dL$vsJ$X4~cr@Qi>NC{&F22`^pCn6=k(?~T&rr!)Y4Aq%f zU#0bL&D4h*)IrjQH0y7m;XBS-cfzCKs}SK~6}nuHi%)gyfD%e4aDw@;Uh*y7?3DBaB;QxUOU^{Snc;66hop-z)8RX zT0P7!#2AAhFjMT|?t4C0Hk^4@UXYzMTP?`SM10J`e1fIHhK3sE-o^*<{^R4SvS(FF z?ldhuGV-3lf-CWu3+5~NaUu4Kd7YV z?o&uDdUQ4Zj)07I-Tmp!&AnM}hhwA~o@!vO(VFzTm9$M6Lv*Cnk7`0x@g(T%9?Dlv zj)&9MBy6($4T_N>0_GGQ%yBMRJy;Lb&al9wygP*X<>uo9=_cB)iS~1cJZrYSStmkh}CRc-rC^>1?N1Tg!al!a# zPm<`^NP;#V?o01WPG%q!kdu*q$;k+}S=tg`>UX6Qa)#+bv9G=387xvq3c+CW_)0U$m7t z&a^YibCdI-$lv^8hN5K(^>+!b0Wk#JvS^;d6M5>tR^!@Ema@v z!N%mSU05BfIU^qYPf+RTFQ^py7gVBp0hM(07r$C&`lW{(RU->p+}>)8%f~U)13V?w zuYP%{#t5>+*uV1Js*f57U?PsB$r!UYFKSMWeU+6Nr{|>Kq?|W`06nEpo|@kp08fcB z>WTp9DWM6=3u5123>=lPek_E$Y3xP^cuLuq08iXJd68KDE0HkUKY)N^AI|V!HTroxex4%XM&^NhMn`_Zdtd|MN9u0f)KKUU7p5H! zLa@B<%(rpoU|(h|4*ylDof>H`|BOq>#%kJw5av#;uPidRdla%R-s1HvYQN7);;S#k zqx-z+D_ut*H@oODWEURQA}&@}#F(d8FwA_TE6M1Cj7h;uHxo<0Jd^(se4&6 z1S*JA`r~A=T?>>}wn_=I{=wrap?oslRYlvtj&Kp?-BQ$xr^Ids^pw0_Jtd6In`1ki zp1n<-LT0I5Z2a?Y^27pr^$C*Hf|rcuJa9V|fBgpQUbr zo{}5TQ_@mJDkOUKlp^eEy9ZqvUOXj#x=*!j#6VAp;?+~y)(3b>{XkF29`^Le75P&y z%=e_2L{XxW(8|q~7EazmHZIR|NpJb#NrS8R*#W`wsg#ZapGR-%`u6neUOc6D<=nSG zPf33S9q1{6zj{h7zh}?ZCpq|wVn}@&7h$jv{Dk`fo|0E;6^dJl0C-C3sc*$) zdAGhnq(g1)?oIZ|;tTb{A1A#-P#q(>qn5@nQHPXOXWW!J2GnFQ05us*KuyMqzXwp0 zNenF+Z#skfHk0fPsL7aJ_}xhIRlObDqmH8wTAEz$ca9Sk`S)+I!v`RNy?F0{N@xYtNkRt_LfMxrFX zREe88;&G$fU{4nkCqZ8MwjA>|tz9rsCvdWkU3(w-E%hmNAPZZlw{>{`&VD0$e=M;- zhJWGIV3yRDlWK8mp_@8IX(j?o>b?WK0q@M4#a;~!UG^aDGD1}iov(Lu=(@CsA{x4- z-E`$}W!R!EXc|>(u)0>h?`+A zlpa?cv7sO?jH8?x^$=5ntxFJVlUu8%(*E^@wM0Y3DvztskKbfUYBOqRrg7@6e#g$l z)o02&m_*C9iZG6A`plmZ9e_K0On?G?SRZR>B7481odJ@?oxd+dN4m~uGwjdUQUX(A z`|MA9JJBC&7LI-CRUFUh3tDJroYbee;dz!zn1zNKEk2E3Oz`aYL-cU6-#|5I-L&=_ z8^^-=-sbuj)|~PhQ|lSYjeLvR4%f7U@QXh_diQG3EFf>@MR+lG?l1-G2qjZr&OlpHI|)*198F>42MU zONP*&y&|KN-tv6$m{M2H-@^TLdYNea!R6w5x9-{FacIK0pFyvCqc=&^;_=rl;Pw07 z>~(8ugO?veWc8&%iYGV=8&&*^^dd#vW-{Z55l8a326A9f#@G2PFet+d2+BnK z4a$%kk)J;40E04cfS^n_p9f+0dJE;3OiMR&5Nso2NhFo9yxXlRJ9k!cxO}pUMqhdKyYgWH?x+PiD(9>f z+R{vq27_6}g7p(GVLu6?GS=8#yIEEvxh2V;pbTN-*9|LuS);!}8Qhnk%=~Lm1{WBV znfVixsa=(i%dq@eKF42;v%vNelu7&(lqteydkM;1f+PZiG7d@mUW?uqM8b`%A}>Li z89-3R;|YeZ?&O)oKnwe^iTUaJ^eg4%ovdv7WjYU_h_UYQ{<+*WNDq{GY&)@qt0nAY zrI++dnXh}F` zs%||Mg-jodZoJ)+{)cS5pc6s3h9$ zdq&yfzf7eNps94$N>#}WS7j)$oUZ#~D#Zg#B_n{TbOkh(>Y|^)pOJYJp?!-t+n%cK z?xV7^O@a{5Rr-l`M@rI?$6`(;LH@i)X#LBHXL{#tiea_)OR@0oKJb; zuRc9v{e5GLU2Zl{-TCA2No?2_{XA{ev#$C=YjUIy?|Z9sjUg#MQISZ~j{eY1_Fu@P zUYMiR8W1q*U!3td^&{jS$&ATMl|r>h`Mj-|y1+B2P5NnIq+(NP9vtEA)irNiZWixN z9BvU^fCCt`mjR6Rp8-t7b=xQb|E722*U`xO918MzDO1U-Sh}X9oo?;zKcbS?Us37z zm#Z7-|1VJ~4k#*ZOtP5S9Et0YoKcpr{lB z5S2h&h57s771_d=XpxS_KPjWCX+WTOY8Kdkqckc1Axwg1^O2H|Gaz+W7*EK)C~V@1 zENnt39~s{yz+Cm+kL|9RQmtRSxq0q4iyb={5Bw$#&0y9&&+cL^6wmx_C(nxmI~FI; z(hqmsd7v|cqmP83qKic{et8qdk;ieo$q@Q=~v9s^PXW!E^F? zW|tI%jt`#`LKeCqFz4EVfVee<=Vak8nKka9jPtBQ1E(aHYHLS6gF;y%W91LhiMo)R;)LZ2`upTTq zwx%~ia7o$fMaiCHgoQ9Y$3$6WCMV328$(t(V(;>WMbGj}Ang$z+8yWUDwY}1UC`*m zYg2}Pn>6&v6k;e;o&2d{ZXk7_uL)it6S z@z1L01Jl3Vv>o0D$v*$I8yU)=x*fG!xVin}!SR!XL>b4PCJoaxqv)Ga7MxK2Bq@Rr4txYQ+AhR=L>i zp@iBtL=kb2qYdB6LS~yDRg~fEnr0B6J;HH1jZ9DFAKVS1dA9#peU9T9C;~yS# zuf(vUZ^=)kZJnJ8A^7bXokVLxgoB40>A}Z^IEyr{h89EuS*6riJKKTrfn&81Bos+o zAC3pzc{k7ex0gwDl?|QCBq^AZg|+g9Q=uSutt%NEq&8UgMhb6P$=oku^YF4geAS~z z{K(X!88Zb3As}n|-{xo6cQY@;#KS%ydNBHklEul#Xpf*+os_%)^io zN+0f{N^wGw|*9R_1@x-S3~^AAf!#G>`D65-gT< zSTBLOK$I^x^w)_&NV!<~mnI3r_WF4AFn{@n)}K{zUkQH%8dYlz6clmvrCllgw8 z+YG=ai6QSLr@S+Z4!_(>Jj0|3W`gD&^rjN*>PPHVfgY8RHpZ1bj#)%9-tMWdJ@8O#PdljVREtI38_Y*xTW-Kp-#2}a$fN% zA=AAeZ?fOqzkQZ+qfNE2r=cX8-9$IiL+3MyC)xUmleD5w_L^@ zrM#0VbKs8qB6#c8(S1*J3x&Pt z3m1vG`=P`^Lr3WAMTOT{)E1G2*rAFy5V2Gr20?>taNCFPaN!;ZFcZ_+(vQ@nQ3<_PW#WNV8H;bo z{#z_s_;76Db6 z3iUx~TM{ZjRR(Ym1W=Ws=vrxo{#R9I*y)zBe@&ymR|9I=-w2QF^sRi!Fb}`-0xK$- zy(!CNOY(#-4vKr6YZKLIOT)U!78dac@aQ>k|4IZx;=Ma@s)Jn{xjAmTq14SG4d9JJGs?5E9}#fzh=vB$+_7!CzK+IJR- z=y5i(6=F8Mz1uEnK+4`HrN9FetBN{VBgPJ#PjDv*IQ9iTP=6Z%1$`(F953q4`h{E`7ajTAht;IaXh%^h_Hq3CM>n83i?bPk$KV5!L$AA!#L(i z^sEFzq@=dWAdVrlk-kAs5Zd%6Dp!%?AsR8mV##`5=K=tGX@%53@>7=nmsQMdAZ0SHz%EGP<8-y z^WC5TL^zyifU(x7MT}5Dg2~t9VouR`qYwsVXvS&FJT2Qq$OjIquHUL43S_uee)8n3 zEn-Mcwc{!uA=}`zR8URrbuFk_D_TVf7?UJua`v^GSpKj|oJG0F_;Vt7HbD{_8)iQXzm8gM9qMDsjHBN}>b3gFsek^xD{Oom5cpl&Omq+v`T{->lLV z#V#15cxYW(KJq)19}`J|*^_id*uRl6i5y3KS(1qU%PLjoj+&6d0$C+{0IQS_!cA0> z#S<*w4cB`J@aC1p)HdW;E6kabbnfj9U6~s2l&#^JQD6_WeYNDw1bh`H=|I`3bTW@D z=8S?<^wXBzs1c-c&hTK}W9`Xf$1T1ZnpRT;P;E{!bZdV|Wa2 z^O>INyR<~Yj&ICilfh2d@Dm`0aD;~>N#y(S9Onbq>OS2>7s$AVP}3QSWPFNA_-*GH^{qL1--92r5cpBBVm<>mO-kg5# zg-$l$z5RgnkN}3AVQ6EL^W0v6;C97`o7$4iDDR%slzobpmGA@bh~?2_^!Q%3nPl_1 z!z)N8XTt_^S2O?$QGToAL(fp#*#KCOrWQ)1T_#u>d9pJ;NZVFF7vOyhtT}D$?S4V& z{%w%gv5)-odvCVQ=RoM%C%mCs?`N}Tv&YAswQ>1ppXZexz!C;#ea1Ifb;oaii_%5c z`yWe~6u6o{OPE%`5(bPtIYBNsTDcJ1qi=ljgRgWUgMFrZTzszs33f;k+R_)~YM%H! zl;Vv~pN!TZ-f`HDL`v6b+!25o%u^v`A$xvybiBy%zE~G|KIsYT(!q-KBZ4Nl_T7=~ z$~G~$YhIalIBGs}x4D&VEGMfyl&m4hr!L6rj>KH6%lCNElEv!XVdu}5Nj96~Do`spRcIZF>jmlUKc z72=7V58FOu_)n8gG=qOnVTS)Rh4}`U!WaXlFsCn5m>9=@PhnhNr!WYBDU87o(XYQ# z7=Iq}e@2DJ+~g>f`b;@mdO z2~b3^?hQ`@-6Bc68Nu*NrDYmnyyC=SgrcPnJo4x6@_%lPUfdMu3+?>Xl>u!${M=2n zC|3%|%a~JRyNm$yGQxmbB({tdFfWq>E&8$LDBzIl<8NSIMzX#J$80kCpw{u*5NMfR zWjUdvwsbV$KNsyaFGI)6#JZ9P?KPrS=sm9DCPoqMaeA{05q@L*ak~(@KLnyF(K$|S zVARCMg2JVtCI>sBq&@)IZx~x~#5cOO8{gs@oGg4XX5;{wyo#=qXv1aE#li>vqvmuE zt?9XP+FJy%cDO;w&tH_h(cjagJ@k-E>_~u4+n?aUgqWHb9S(8Omidx^nhA2#(Y%dj z&u?xDRq53eaXtylY!VPsi%?@#{|?*mc~w*nWJaU4I3ySgB%plSO0YMwG5?0V>Vu9^ z^?Fi#R7EB-2bPHS^Udu+VnP+gZRGEw~AXn?#gB=JIRPeK#T`NO}8MM1mrWlU4(SDOH?&PGf(7;IimtDBK|H zIPBo9Sg)LQ@0$d0a;~8s>yDPyB?%Qe#}4iKkB;pbp*?WIMvul=k=XUn8=(S}b}c8a z9DjVJ_;8W16xaIVOLvh~Re8At6D;-aOccbxq-21vl&a|eyRgkEH5FurkO=f=jPOJJ z#@Lv(LV4h0k6j!e)$!;ru`D~Ksk&4+QI)UoKKk+YXYYD*I3YLg27S!*i^_4n2hQrS zh9a_fL+$0GxGB6V4*f+(C}!}3zc&s%`e!B%4w;G-hc*YmS1R;%(=-+=KqSP^>a5s| zQ}cs9Krdnz8vLF^Zf(#tsH=ZXyeC@Oh;+L;BWeqRtCzsQ@m|*^L6$$PVL^nHAw*0> zj;?t;^_}RdI*RSZW1nZ+F;%I^KAN4z+0k0j&Ri_9K=5_NZAoW1t0MNn|K7rsGgS&5 zCqsWI5XQ2PBxC8cr$=>b_e6?j9N3QG!AcFlyns!!!@NL&?798e&Va0kg)y;U zsv~6DYHF7(QXfrNN$VyN=2wn%7aaPI%3aVAJBN6u=%zV6bT%gGrWfOM26{SNQN?}) z0Y(oE(zB2Yf-%AEGIF04LBJ(u#@WCsUR(&DgPN47VL4r9Y*EV2&}+KOkcjwH3hCV5 zj?ZutVq!**eh&R%$dK4o2U#{R*NCeX_Q<$Kqo?OTzLLuBZ{~6gps!?8&`QNDZ>c#G z3-Fa(k6i)2(z~}5$v|Hz?exGoZ9kl^AS9V!n{|E;(nCtM*W5(2G9to4PrD=vF>QNg z#pJ_txeVBondH}0jA4fJU(?=4{G41R_ulDOcYsTHZO*5FDNH0_3PS>%!h|M}=uQhO zzD{BAJYT0UPSC^un!@P+-&2_Ae@|iPCOF9dOkqgm%c6h27yXI+{gEDOQrK3LNxqdp zmI^S1LHplR7~}n|Pg2F*U+T-fze9y+_Nc&NHC`1}^zB;l3>b4JjxO2Udp(EDI#|TB zhlkodrEe%SUfyt!YCw8Tn`axezM)f}GcY5)FDt%K+>T9{`tkJxkAn4qq$f(Ye^5~i zt4m@3)b>})zIdEYcAn#?g;Qe^1l@=3owRX%Q2Ned68b`zdV){j)^yy&rC^~o-@w>U zEJ~OukLI2pudd#1%g{hqSBIY9_4={9Gr2MX`a%6n+~{3MZ+g;2ZfAIF$^} zy!pit2Bv}y+c0*eo~}Cb)9k~~fwzaucpj(?Go6O< zBvs!vzKT+Fn941vh!u4!ZSd?;TIMoV328s>D=M6_4=WbT;x3Ufet*+VB~jq@yF~g5 zTB*jI1uJ8v6ehpqW*^3{DQIS;$)Ar&=BUSE8^X0!@Oyi0AbQ~H*KHB7qlC0&RyIu~ zs8%>jo6fcxy%aaK zv?1}qCeJUAU`H(}@-6JO2AxZtMQBGoL{Z(O{ye)MfWhlpef@^Q7c~rzJh4PP4pVubDmL6^;6HGf_s(6@5t@UK*ysbOj8C;5;sP`P-?oT;X z%U!X$Dq)WHoIT-Fv++mxT;`jUa`c_k$>G}2j*V+*D8L8{PQ_!LExEyPbYhlE`%&Z6 z5>>Z3CoCDn;UF;|Pg2{wH_<(dh{{9^$mSXeD-N*lt+ zEbN;XnY~dhh0<&~vL7fFtjb%dQq<4gV>T~gcN`LFU8`kr*tcT*j@+9gj;U@>ORxku zUnBU}_`7w-cgX+3QHrcfhEg6Uo^6YG_oK@D-s5@9{fDm)?lWE0Ctr1aYif-q1hKehc>L_~GW{?Eb#{@5RJG~te znQVZq1ke3m15#cN5-bSGW3RP#APT8^Ss?MV5wE;xl0|GUHUH8k%rPQN9psR9T1{*( z0nFLWs0iEW{1=b8o^|ym!r~Z$hkNnB#_cRoOqRx+o?rx`q^qvqspFEeww|l*=?Eoh zPU=MB+2|N)xJgpFTV@G^ZxYi-m(QFUnMcbk=zjZgXXf5l{B+eF+N3-(?dP67)-&i# z+gb9(d*+qN=J>uyt85Izn{B{Xl~Y4rYzu9cRv4jY?BFs-UVsQyq5$Ty&(;f0EP55m z#f9@0_t#aSbdirkF~R~`!7pp-Pf9~}YR4xvy8Z{SBv8SZr^3_Us+dL+)HshJBK)VA z7oxX@uGq=KaWLmMbr{(TGzN7nRkBeY3B!EVWr$mO8P@kG8x($07S0gcnp8LP;|Bd~8#i4yTQ$@t$)wbnCNyswwS#`J}%i%~mP}eCU z5vHwl_EAH}17}B!)t*c)kryOeFB?`+c5mO1M74{tJH9T9($gBc+2{XQHSer`K3qsb8u0I3Nfr6 zs$RsC$>$8Kxs(xJNS_8wW&}BT112tixG&(|Mo#n>h!sz!Y`BwsgIs@jD3yU7*(B{D zTyEp95f)WjOPE82GK(KH^Dv8fuFLy>?A>Khoa>^e{h-0!-QAtw?(Po3Ew~fh-QC?G zI0+Km-GYVS?ymhdS!?aH_daLlwW+B(Q>%*a%?G-xAMXEs{Q&gC(2XA}qX&w4P;|DzsE=KqTx4E_HJJ(wT=(;ke$|C1h!)j#!MKK-xl!I=Ej zgK_?s9!&24{~k=|f6;^a^_BQVH^UvoY5W`Aj@9QK7IlFD`p|2y4t;zDotXn8tvnCF z5F(8AY5VrK_Cn15U=Wfd%3i)E#Vz=l%*(qp+wr~5lUKNTl}PoMpEDglIY?;HmvBCc z0;7iCiB~GXmBuaWw3Eu349Hd7gXH8O(QtfqagzjlEyY)zaL_i`>U}TaESOolM-+a zuxTHyPxwAQ9Biycp}0$;{$;+ops6|-fJcM zd=$s--4shTdRdKA7oI_`2&vA6E9#oC6Spmtx*Dj<6nn^e;UCSoPMG`haO0I|*1I}W zJ{_oQz^Q>HZ>}Yp!|PGs_?dRUqJmghUxepfO=m-|-kGvp7tG=9?#Vp@4wl_9!{)O) z`9dZ>eTVu-pKJnk1m_rEeMcu7UhyI!3=2x54Y_4qBkmq(qa zH$wI3u*sX(T6mnw#%^Iw&WEVCUMA80JYl8gFjZ;|=>rI{+qhd$D; z86vg-+|=5q|22Pps<~YEWFxs~(!Ma=$*Pki%O(2;leezW?8zgCT#gv-~p$bMsg$ zp+NBeJqH60QU&Mtdkn(~Ze)6{)35|1$oao92XhO|!H9f(_g4<41K_^uwR*-k^!72o z_7rP{^L3XsPCVm7C0Ci{Bw|B=8Pdi|^%Zuyq9eR(0}1|?h9M6E=&h zsl^5eED2&tfnZ>EEQSw(H?ZQm%U#Tij(e!st3gUuLf1n$==gGw$VtSiM$dH1YVQl50R)!pSBNuj&v$}I-hd^wMDpLj(j0yRx00fq5$6fTXl$YBp__wh1;bNG2Ta*$A zKB(v=-BM~LVSI?*=b}~pX64UL*!*c{`IMS{t#=QBzhw`=J-UN$&-%K~bI5)VNcx!hbUqnw7j&b$DeynxpviU#)tGYCE3 z7_~jX!ItOiWN%?FCE^}{fuCoajY>w_(3EFYi;X5~*8vnl>x!ryiX!PDhH81VZis-? zMmjO*{3_Ztn=2bHE`UM^+dvsAfuj>~W@w3+I@^$atphKCY8uM85YV(`tam^coP$S} zph7h!@p;hK<@KO_4Ce4S4wWiX_|?%-zM$1GMj(8Y;Tf~r?Rk1VlHpmyhdJB%!FN0N znC0ZqUtDO2n*iK|=ZXK@NcLx6G+nMIAMP!{M1=e1F`%VE)6 z7hYyCmK4YNi%0y+_ywa@npmUPS*Ta^WXK-i_~xRL(BP@TB9S{K7tmYyy4AbmBRQ6+aZHFH8jn9_l4&j{vLcpL6mnHqYZUk!2W2!%&E!Y zPS%DC`QFh&=%HU*O`eccW?i>cFOgcUPzf6Yk)T~_ys9eLUZiLj+Z;K4DLn7}cA9rx znahG{-tFqRnv((s$jRI@<@R<|Ulcg>XwjO=BDn^JBr7u%Wg@=vlEqO423>EeO( zip1&&gQu7))ImHXbRwDBwIDSTgtN0_pGf7X3R314W(p`M+xtM(pn-%->0v ze@(*p{kKiR*#DoIghBl~3G;Un=Ib7Hn3^d$BvEtA{#Yr-;+Z? zK)?_D@#SCjef9Q50#}iy=EIsYR|x1WRk^kK!ML~V!E^l+A(u;@^aEd{$Mg>N zt@IHB>iHoSJ$B-uHnI?T6O2)0hc(ZIQ*8tDz1t>4^TDMyQuw~5Hh%yWCzXFI8^zcXc;20RAJ(I5bbt^UpTw&DhnXm8Ozvh{WqIVy9 z7ijjO)vP2*j+0`^k?RB{n)n)H9bzz6>BA85^P* zauf{-e4%{0eD9H4;VOy(ycFe-GeVfb-(r@jCa(Ee7+2}LeU<-Y4eBVBQ$6nJDjF*JmZnsQ`JBnX3_%D zwlVUKhJK<&=_uN$&oGcxN`w7%+4dS~=aVx1a4+9x_q!!*41q1oizdL;5Pbw>KA3_M^G5h1rX%&x>Fe#kFMvfj%@VmSHP z`?48$TVF>eW4bIu2aCBZfY~$)vWon)kxfCQ=R5vRdlnOmf(RT-bV;k1Golw& zVZ;hkc8ARQEls%FfizH<+K0+~V#oQAX}$AZ?R1i|784Bc$Mb384|o!C;p`T5`q>Pm zg=^5oeYQ55LX*e*LG|$!|FD+g%w)RZ?lb3SCrR^Nzf46IO37apb84OMfb+tl#N_)j zU5CajXeY@dlL!AG$oG_|XY4C>CrmfGF{_H7&?MNV-rJ#|TC0<)b3|gcV>5O>^Gy^mzb<|1aRWCUbbvA?>~o<6;hKJ`zPA3b?Kj zVuSo#{6Pszrpr_OCPN8HKk1LsOch*~O1!)9nZm@>>ZrYm!|!!XIB;FV0u5Z(1XEY{ z3fGPHr(P&uwDW$*L{B;Bqmlk?ElmQgrFeg!wX~T3t+HLZte4aAbN*w&uxNv=nhX^B zy?LNHSZG`mSDKC4h7p!0SsJHS9wHcNZHzYk((rCU4I(Ku7ds%!t#LU^YVeyGD(v1~mT)3Y_{!Iu}m0 z&0XA|Znf-wBO-RMD@(+_kqJy4+uLX=vl0E*gxrTZwnk{H86ifNY!JR>t-eH15dDdK z5^U*pTpdXUnIJ_h(F$2Gw!DfZ3It=CtzpSLBs*hx(;jn}h8a+wXekba;5}4p#+k75 z5B+z1$(|w@e>jf>d=y=EJ17Si^scT#A>HhJbrCXdiKxb48d=U9ASi6PF#RwYH5a?>j%WJzr-2ua7;^mo1)2n*kPm zNyy*4CCA_157kRYW5dBCbp<}q@a)x05-V3!W6yxQSxBBUE) zbyo}YXPd0$EA!Ptt!AO@*8kYpsQ=#BsE2mI12;C^HE$c6pVn_1o7uOGjnh9iHju!L zjR$aJGZoSs6HvI=x1;?XyIAOE#~4sxaiFxT~G#MS#d)F1mqcB=8?8?UaF^*$bX6;-oNeSEoXfT|>vaxB{C@*FDkzbU? z*tVim+CH7?&S*QZ#4z`L^uzf@Z@n4nx86)0us0(K?9EUy7w-Jko4NU;H-pC8&EQ~~ zqwQ9$Z!xVR;f49mN#JvVQ5x%nz-LDqaMOARA}w|c!#h8@TpKiXhE*0i|o&W*3B z^tb)srAi=sc=1=YuPK z37%@T`FOnT5v#(>&DDd}MpcuG9a}s(mt|#Td5Qm#He$2u%1#1qpqX5kS!tj2tlFHW z-9E;sO|PxxuIh^62hu31M27lpb>6RgHXf)@*3k1|l>vp#>}5E$nZSun{=z>dHgjHe z`q17N3R*H<)Y9uh@oPu-0IO-L)xa^C6GRc7pS}Vf%Rp(#z10`+dZi8=*er8=>2cev zt-BqWd8H+=^V=!%mg%Tyt#TqD-w4BkFw5qqwUC7U;0V>7I$@pp3?*~~?`)~DU~C)I zMk19IHY-OE`5I?$*|@x;MQaJ{5{dZ^&x#F5Z1;-LtvAztz0T zUf>UJDe0fQCEU2J+(NRbga)4C5jHbQ8yYk9#^#TIc}w0Ph~LQmYj3IJ&0A`G^Ok1m z(f{EsLFoxm1pLEWf}!2Z{?l9X{5NlD^e=B|605Jp+`}2U|9C8_oek8;C8_xSbo^q} zs&I?5c7*X#%(2fG*lA>rh)jzf>P5XcQL6XWoD7({e+gO4_MB2&(c?Ck*jvto8$QHg zMD@|BeMox6h84SA*~1Vu)8-+$1mea_nFa zncMqv`&YQgP}0`hz{XL$H#foEBcbFx2|}cE1URsnaAb<*?ZcGUSy5=iB=8tAS=bLf zgiT)1_6GK5wC)3+rTH34`j4@sQ+rR7fxVfiU~#?~mJaq#31v$nd4=^9sr)w2-{8{V zZ*U35>ypT&JZm5oOB(cptO(6$Lu~naY^+I6QS`QA&!nl$+afEOz-?eUatz34`KA7ss32mV{N{| z?odBP%Gv@9zVs2VS2LFVUCWmgj347h8b^c4DH=umt`I*)MB*_tX_-fl0<1%nYJ(PR zet=ZT)uH7wZBU6=SKmER?e(u0e7ps7jfby<8T&Tgv7wrT~?p>M>6}7XYx9cV!zJzMdbT$gOma< zdyKh0{2nWnQFi&a+81C>p3G&>ONB;hk-*-J$GQ$g{XVcaLs=7q3GB^`|IwRa$bFDc zL-|A>>yzJOso&ZRVy)R!wJhtR{NA@a9kY^9quQkyzb{}i`Q$|Y6Dq@e&(@yOuAX3~ zb>QNky%|cj2+T!bZ^rhmHxoiJkgI>Kg?!B&D>U`HH)H<0H?iusSLYjq zGRh1!aSJWC6WSErGzt8+Jl;J>fS(51`rg5+tfLT9R}xzjN9Q0Nd3#=ez!MN$LI#3M z;BVj(2@qV0^rkzZW)a$Zrdqg|~$b-1|VG|5(^i zA;8$3wr_qbD8v+R11@Yzd)^i{>@hl80_zsm{CBo@=KDFt7VLOct4Fq^^~&*;#NyzV z&ER;K)zV1wS&(Z5qSc>MWjB!dS+NxjpB+pUL1E3P9Tc#~1GSQ}u$T0IiI1mLT%wVs z_S~&Xx_&Z7^Djd7IA0lEq&yS&4A?z6QaFq8@qQ^@LLzwKyFU@o2h`VNU&>%zlrfh~ ze|%bd*1PGZFUPro8zaV_5Lvsc}Y=X%fBPhS{3L zT!EmeOW_Th{TxRsvsSmeP`j@x2R!AA@ar{Y8{+z`Wak|?Q={BuFhy=*09>`m!udRZ z0nO;b8@NQ9SM;{9xn&tA8Wz!Vgm>)bEt@2RsI)UKUA=B|0^EbF8hx^CErVRCHJSIgyJI$apRIRTf zho|y~5@*gL4Fyci3e{IV4C@XE9MMpl=JCjn028Y^48K*nBnnq3T%cj=uO_w+(#!WZbagn**TQ zA8BVHDcx66O;2-?-5v0NdA~XYh^+cT`q;wydiU#MYw+fNJuvXa3vg!ZU9$&ZKVy;= zgj{~E&?s#|eqC|z`?(`h)tLy`dJoXnzgSnBI$F5ftPE}EAo?lZR|x^%73ik{tGW_{ zar|RqUFCdx&Di*4O6qF#s89YdkFPw_`M0-J$A)f0UR7t>ni+)70lw*rymY^NY2YUI zo#El^U^dY6d40?4;mnoajRa7B-X$)&eye$lL2~oKox>7(u9rcG1h%l|*Yq*S3K}?3HMb67af}o?CXC zF5->Of7NE3OKLO!r8e_5DOLb&b|JtugSJCnS;K4!-OSbw@e8Y?tvboENr-?9tBb+5 zWjv%MFUgs~$KSOXbzp5~=daofX99rieZprQh;D`lyZgr1*R6wYdwWK}+Q6RKnLvf* ze#WFAj^$z1rlFLVPuwnDFx0wF_GDR5f^}VHjB}KM|4E}sl+BKS>52U1irv~4wiE%q z-p#OFj*F!2LDAg1G%{@4hRifzs3wA9i)~UbJ7v#Yr#b-{0^cT z8LNj8@Qe11zUTy490_rL#qbq^rN+=&illOVx}=)SDtAbw(bA%`G`-}zYjR$rg{CH* zJ@*UvxhBg6{6yAvq>pVfu94U49>BZOCFX{cvMZ&Fm%>p>@ub>JGx5)6JRkecHq+HB ziSl92Ye*%kkSho!+6|MX-*VCr!jf{q6)Jv)cOzK!thqb#0R zwrxJzuj$9a35u_RoLqp+O;^n=ekUY4pnlzNmhyS27~3m=UdIxpSxUCTkr=ewKIw^0 znNAwA33i-WBjKaZJ8hNq=@5BUfUl88bqL4VD#x>J?lOtkR`A*edBSs$U7{PT5^Nmx znU^!PM3)>{5l$WT_PB(RArrnB*dC)C1aeD5q>?eC1&spb1&vddJtdYkD{8P+4|Oit z*53-}obPgV?pMPA=DaFy-{J28zwD=e0b<|Zz2@pXc*DPTg&X(|s(#mgXaj)l^i@B; zJi*5bl;j2@m$F5KKluz4ZCA?p1ygS^Ek)@Y+TIT^ZSUVlHy2omS4yiA<`RrX41Sd@ zHu<8J0~r+@!1GOQplx&AtpjkB>+3tP>EWDG#Jyr4Q*#B_&dUKfxwiuDU!T_p%tP^J z3k>7_mhnEP3wPYjr1BBul zFZkqy%|pqK;VKVK&EiDEgRyX5e7*P=lI2nNsRc=}1ADy3+@3W`bLAu1%qI{$piNbV z3=DD-m`2*Ra>c|5Un76uXo?pWsq?M&6-MAztM}WvxINa9-oK z5#Cd_E)W^9^bzzhlZLPg=LQHanL|)UR{aK-j^4l};l!yb1^Q*+yrvX7Kl$*Ndk1jb z0=c6k&p(IRjG`%ZEjFPEu-P>Y*tkL8J6a4q^lxJ@Z#0zqVb}yBcWoVq9q9-)(u;fTWN&ft<@{)K;Q$f?!>`QqEW zCJZSd>1|%af-}}vlThWX8=A5B&@7qLye81F=6HocjR z_G1_7oX0B8WBcVXboqYnyn})5R~rrhWzb|%x62r?&RftABwV0?M=vU%h>HwEsVGnh zlhq@wA1 z(%~m=5G_K%HV=2A`3E{Jf_YgxnwV;`Y{UVf#QF6|Q(tlo-K9c_BDxKpi3x; zY}a#yIjZfRWo&X-GLjR!2N1CFqTteSTrn~0_c2a=(CAikBn-Dhg%ZkA+Ev=ig%XNE zXjA;-*twQh>Z)6JVbGsR?uG&y@aRpBodyCx%PEW>*v9vU0(r~I$%xCYNDRBT)YV+z zc7=rU4Qp^=li5+E$t0XjxW6GK7`4yAM|yV;bY67H3Aq30%`D`g zUtTC9{?VIJmZ#dYL;^A0)QhJJCea>v)O4kg9!k=Oa*p6pX9`L~)+Is^&~DOSFBjn> zA9c?E(jtr5re0yITG9CNoy0LG)x;3)fN-*^Nc1qnH^cDnWod@;i4f6S_!W) z?KSP71(7G?j=8gv`9WF*^_;`VRO~?BPbohE{I~uWsEMYvcSI|s*gbef(4kh_tR-#H zTq*aS*C?%-!jMrqT3=+b$WK@3`lV3uCl2SvM^*TlGiq$39`G%w zrub49N^nCbp$TGI3p9;paWC6ZzQxSKwTZ9x-VbP8MyY+B(rEq-bo%PJz&{pfO9IDs z3jL9Ef3pj9+I~u;O0E>qAbjQM{thVeVNGJqlJ@CClp5X8ci=l3P3fApc<-2C>dwNt zl3IG&NCvGy9mVvS3AT}WxhOTZZ_GbTeiX0C0s?Y~@qoJuU!lM+DO>~F5#O7ttvhJa zU3!SRL8M#g$TXC_L8auQjJVq7sWd5gsQU$zmY8poG|2S36B(#KY!xE~6pU%66uqkN zqC9=Nu4bNfD<&f)S_Xc?c}K|rGe{bO2ZY~INEtgX{q*C%9L11V_$H`@suw&auh!ov z+Q!b~^m1*UM>c$g_*^6qJ}c!b5AkikOF9sOmktH#&JyF{%Dxmw&0^L-y@ECbuclce zqOzn@c!MrgR!_*6J1$P?|*MFzi0=O21=Nr~#if)tajBXm;5-UKFQM{hpS5jO z0J?l%=sME}@q*3K!NYnNllTw#AHIs(b{eh{Z51zp27Hbx=HK3K5|@SfEiUco4PNR( zNx&L{bIS-r5*Qrpf*yEE576hU87t~*30Y)OnCDG(k3$B`=fpLGQh@h`;a*bB5IKuF zCbS@J8hlFO0Bui7iU>hs)>v=M(v82cGa#G5`JQgx{86&}2bG{U##e7a)>e!e%ksdK zx8BSHf>O<2y%`t_=@Kt)V{TS?l2ftfv_9nAd0Xn7ykQF*hzMzKV(pE_@7I_P$l;?! zY_L*vjt-{@Y^o$N&udKZe&6Y^B=I2XRxD;ZNw)UhhfW?H#kC>zYdg!@hz%&wH`XP} z-@xbh(v4TqJrX=!iggwSF1|CXc4~*~^uBly%Nyx#s__mdR87)&rP7puX z`pMSCY(o7@j|v4AS^B`WtPOnv)Tw$pweFPl#8d^57GtV7ByXy*2V+1h_d+uQv%hiy zjiPD-T7-HnDw+|FR1|nE#|d77oq=e;UOAew?iW47Bu8}n#oDYc{c_!Rm!Dx+&Z~RA z&_q}H9gM3|XKemBVV6&0ljk^L5+q<}jcVtX`AsfaalVrRl1r_>{vel@Q5Z)K3vI;d zRA*&)kTkODtdumPYvd05YXUnV>=^pW9>btRH6<<7TFr<(J8HnPi-(Ri4^?I+23i{J zkTJ=4WkiPioG|jHt;;1fhusg~$R+ty;YOxo*d$rZe$XUuRku%_5*&1X;C@!*YT?O- zU~}_H#T(80I9S{43CxQghb}OHPk~GT?y??jzx&fEr!91}(WvBKE8UR8NHeJ=vl(K1 z!iAB?`441RI@Y^{Q$#Sov{mXawjoT}CAJfFYHJg7sD zHzr|I`9T#zMN_Wy1|TR1dl-?gX?f#XXCa{vI(t5=8p-l4E9=bdDMQ@#dCd+IM_Q(iOP~U-&{?tz(+z* zw-QxUWkfilQHUE-qivPtCM?1_9l5|qT8JA&t%Oz-SM2Z#F_mBZ!=! zmQRITDy5E(Bis$KI_G>O%Zf;dZd@u!88SE?hwPWIm^f$dMaTC2_uh=_Tpl!Tqe+pd8}2eNB^XYq3pwb1~yCD&$*l1_s!hF;6;+msBC> zquPWpcD2^*s35I!D9g}f#~#c3%w5r*rX9n;35g9|e8V8lP{vLqRW5}SfQwc;b<57X z9hT`Gzv}+ho4Li4_DXOiPvFXsVELmrqh2qtGG)eqiKq9h!=UjDC4t*rjRlyJUc#Cj zY6a~D`KFa&Y50zxJ;KACMk?lT4fTs$swj6p>bYWBUdBtmU#EC8RNBR6e0o|T zUOy#Y*j}jIsWd&dUWyNw7gtrV?WaD8-)*%G#g665kmEm*OTakDN0VC782{Nuex>|1 zM7q8$QLC1ZtT+n*5vuiq`r zy#H2orP+nD2PBuq=a6G}IqTH=R=}hLPgHGBS?9 z*X@y~2Z{5Tg`D!;bodv#PwP8>K{)(bIJiznt+dFx7Rrsm*X6AgQ->hPI5tGBQyCJ6 zEQzy18}Rp9P&UFpTU>TuxfoxQo6NC?a z4)WNvr23=3gEJ|CH}zCpk4o+cyiebH}e+y%!XUS*(`4`gpG z9-M*en#b$9*VnyY2QN>+=Q1F=l>UY;98Umlvva=TCGg{~_!_y~8 zfIAL;9-7H$FqaQSpi}xnlgi)~85To_4RdR%;hA>s>m3X4Xb?Ld?94-->7{$F6B1Qq zEyJ)JH zj(SsmjrF2f<$jKgh!x~Vf%eWw?$T|KV$OD0jjtQcovB*&hb8s3BZ!W~ zZ!BL)kOw7t^=e@o58vDK!NU;j?cNROn#GZg9Dq_L{@?y_~{oUm9p_i;dO zxvi{Jyr}TlT_;Xd+O-1U?;gaM9x0rwgUe~e#J!p*qLmG&^6IK%RKv>$Jf;+WNE?|2 z*FVme7H`_MmMR&1a!&j{0*)ju_2I?I&bD46uR<-b``ybYIWDyh1qGZ{zQ)&|6xUxI ztR{0EV$GHyT{f3-)Ug5-GBDvAbv!&5f7oc@-*VKc(ZBzOP><8(Q5#*U-6sEZV?;l( zgoeUrh>0HKjk|AMzIz}9ErN@{0~4c^*H%c6uhY6BzGILntm z*`*R9AiH$DN>3iWf@vGmJ{+U=xMng*1GeM_e-?Nn{=;yOCamq}&vX?|HwE;)k5^Ti z+96*I`zb#arpvg}f3BXLuSK8a02z?^T83@k;^R?~e_d?m-9`&54C4wuo!Xqe#+L{u z(_}ADPE8JEmpURAKAL)Pfi6J+*`<=g07vv=l3gs_Mi|G2_?31Zy`!C#rZh*zrW|&O z=ITk_o5jgJ^WcVsg^p>_lwU>FH6i(8_5j8F%FC&^5rf z6sN1$1|CE)QOJEAO3=C3&uQ3^BdQ`-C9Q#pSzgGqYX3x_Di*2@;As9n@p^LgqH@;l zNq*ihU`qc|LiPE*m}~78pi->R9+8EyNdWG|9q_VkxdqG|CiXtv<+facc+NG4qj#eE z1~+M>+!|cCgV9m$=p3r`n`U28K}+;eF&&L*35jS=#fw07I_s}&OF(Jc<;i@jC$J_R zLzgNL&H8kyc^w)gQ#j8ssV$`wJ4N>>lf%HmPeMrW@uDd7tuwRvq!|3wtX>?I?W^HK zQ0!Z0rWe9IIp^2o_d9-EO}?Z0cXB-lu#@a#RkotcBT&P?H3 zXD0Jgx#nM;8PRBvQUtz!NUPdt;YUS#e&I`bHDkf=lVJF4azf>{3dg$2M`N z{qnNUNXPk*2t|XqfT}@RI=RE zX)p)R$7WfW@%cYOGtFsDP(#>yn3A~|YQSXo# zq5PaWl#N5$yC!$4w&d7j@TlY7aA>>rOI<{5I#qwoNgf&w*Xz zkMZKu!d!pCTo6*$e7&IYj^F-yptBpx=ixWIWM!*9)qwKzaxLz)Ad0*EE>&*)=M2$q z=5*_>VAXQguK9qmYpnav>qRDy3qy^Csv|X@KtXi7_(4X3V}ji0>)bBy$A@j(T9(e> zW$S*#v4j4+OY_(eBZh&OgYD0D=tBjFl0`YKDEe$-3hZ6!K}E_rt=_vlM!;|-?_)4b zsmI1LWR=l;y%p=c6#g@Wxqg+8Zqq`tD-tZfMso{Kn_R0ZY;3MXx*LdX4ZMD8`al+z zEJMx|mA)?NY=4tb)d4C_29~eqw7J;S9xtR2Vfp%fX|52@PB^3Li78)T{CmiQDc1e#VSmJ)nM9}|YFj)wiF0Hta&y)?(Ov%yA z#LFL-U8iD1B~BM+-HCLmEm{dYL43R0j?o~u{U7Yo<6rF3uNHv&%jn2&cBv4^E*)LA z%)ZT-9v=Gx*DRBQ5*k5v`7ekf%%q7xI2-&iTZttgAX&qI|hcO5kae0<0-6yPG`|iZm!BNL3_gn zX=#)#kPOjun0*N1Dvv)A>&N}&%k;2;eix@D#}HjM4&Ttcnp1GMO-{%*a415%(h^Wp zxlFvx-U#;Vk%wpXhT-N`PqBs5ZjjT~u>H%ou&YA(Do!44PnZ7`nxSe1hGtx~s%`>y z)_|dz{=Y&qHDndIJrq}-OM;aRRX6!w9*uIx2eQWMMAoX);Ets91BvVWcwLv8kp@XupHP z*1{QBemd~d0x6uNqhj_-tma~*xA^O2&)}C6DS)wlAZCAk0=Rz}j3r#$@FZWdI6N=L>9dvXCzh3(cjA~Uj?$6Q;Xi;0?{S%H+0Fl$oLt?rj*>+ zgcd;k8(lggn3TMn zxJo63@4t#4>_a%{DUWJUgzdpoeoE4v;$pe;7e@TLReWBwUk zQu_;C;@X6${1qYJ_&H9&BgbS%0@TgG=pA>p1t;tM|Mm0nj){+&~WO%+4$N)fbAF63bwojR#vE0 zUh6L)IHU4BZJYS~sFpq{hT6#-rdm%s&jvil9$E>;r7xt8UxL{w`X=V7Ex6{8{_Gk0 zocmUp*#uT*RNg8x!@$bS?GV^4_a?A169cTwq`p*SzEx(DT7nr;A1=4iQzU~nqleQK zVu&d}#7o5CrHWblDvMd>&j!iO>b6v3XkHl|eOPprC_gwAXgO~~228q_ociIO1G+cz zh5@;&Prjdau7JH63ZEBz6%gy4y;sB})Rx1$Gfv*uG#?D{tQjhb(9IQYUWD)Z&1kD?| z-n3T2u1zoZ*9kykATTnsky!e4w+8?|0dJ>aK#YE=$N$8LgH91i6%tT|4L@;!z12w? zdFx15KM0-sg^Oa-yHJl(dg4}c;-j`GCR{*K^qD2f2Ocsbb4@un&OkD0y1pO#fR(E+ zHO?7=3Lq%MSZKS(om*WWUYxayxSd>wWZoh(kC#b7N)z}1I-19G1Mu|=O2E+5xxr2U zaTVih;p;Z0R2QCwFj)5m5z&gr_SNW^$F{nQL>O0h>01DD6u zA-mf%odRGYpy#46ZknHCa<+@-6#{W4?&j>-zV<>3@SNMu?|HLj?Mv{9ckuA4%a@Dy zM{PEsd-mn&384O`x>RHpQ1KM)ShZW*`I>h~poNxnwlf`V=w=9Yv^@_l)n^T?%+$0I^2StM)(uU;InB2xM4e6=S zA&VMWwFZP0?{~juO$V?W1Q$!L$^xStOrKNu<< zRgDUswB;V`=FosHS3W2VjiRDB2pi|nd^6Wd)zu*y?;{eHR34CUPaLvfLj9(kg0uHk zzL~-2glA$>ScRNTXonRV%XP*PRhL{zNDbb8+AU+i!83e*T8rcdJs~$RGNTt4r1ch= zQB^y_2;vB=s1hE^WNL#}H0RYr zv+VitE?RLJg6h~QrH7fuAL~@$=Kbq;LXJEx16i!cLUua*2AXH%FNqKuv2NUcJ;DxZ z^?81tAmYOixlu~-$_PBENCfP0l|qceUN|MO`TQg$ zjG2w2N)Evr-Gd-koF8K`!a4VeqW~F&`KfWC|3U}PcN0p9UQ0kDhdO|g(BR?OElh^& zj{+70eA{(y;W-Eg<1~du@?Gx7t0rJnrT||NjToX)!D+yJxgCA}p6{b@?hs%7FFFZW z?5C_Rd4d(gn*4HC+?Pa%Ry0yjhwV_B=dEgL_<_*xSTK_*y? zF1;iMbzjb&&wK%~SNI;+`+C}S(}fbiHH@5?M^ck{uy^}huc0S4uSseW>WBx9s9?Ig z2ANELSjrdvlWGmccnrf`? z$WQPQtd}8Q6v^imjq67lG)b6fDAHz4gtpwJ0@9{Mq2N5@`Gh2t{7u`9OVI8K`wUT{ zC8-j5`OKloc8$zR5Fi*0z*$94o#RE&^{~KdWq-*r1skr|Yn@5^p-RPUbu6RYLeYYD z?ZL`g>PlUNC$g#|>C*)`*>b)cx|IxJdT-g2kMj^^`H3`!ypx~<{InzvWJ_J04plt- zi9-fe$mctb`m|$w!Mq3J7ZOB4JelP$ahPUdqA6JD0sH1`IVq(E!v1v=ry4}ll~@`tj=R7ouZa;;cuf!AKr;%Q9XT*9{7mTS-;KXl`qhG1~|ldGro zR~#pAb}d>O81M}EA>@CMf2lidP$+>S&gy)jltM0o_LvLyl|| zs%q@OC@jUI4wXV}*Z4Jvc9Hm<(1V&Sg*xv0cTu3nJe!O=)%G<*Uo!l3<29n*dw?TO z45NFzq)#-Y#nbKqlUX9$fR00hDZ?uA80Iz-d)p)JT4}RvCP{n08G6{^WVBqt1XN=z z`A_UHDZ}#8%t)rLQG7e-;Tq>fG7%H*)IoVvtSRK>UCC{Iow%Q^$FB9LhoQo55p(zn zw-RvJC(#uj-`DjGMb`4>X_y>;H4Z<_HJQ&(F7qL8vEfr)F>V@Qis5?RtokHGc%ZQ9 zJd>-^&G1+|V+ev63EuZPL+q7zwOss7TuOcumnu(^-^3+Fu2FIoR7{ofZ*ipg1i_wb zDByVF3pO;$BtS@4L_f!p-g^iNK~e;~L-AE&xgvpVK#pi9U}~zKkOf@!cnCS?$XOdQ zLCQ$T-XZ@ch?oSz5kVEX%v2}N#Go>Yh}T#sb-RM|LL0ZzFc|e>gEJ~7zU=<;%U2tG zts?o*7zU+GMl8-Dcgto!Lqg+nL~f62P;7DvBhBrh%? z79E@Cq^Hy{d0xQ`Z-BJlO|QmMrs05zO9^HmoPw(fcGf=3BR9f+FV!iMM9Vs5x_{{2xO)6x!XWwu10Pi<-a!?X{RnAaJsjWm)Aqs8$clRKI}BZ)r6G*yep<|UQEpe2(97t9}li=~O9 zkakzf@q})#V8;LQpb19Pu))MC)`eKTq1~(1puzO>&JBR`uC%X`CZ#p>WL1kO;d1qg zMkS9JHJ*VoZMTS|`L3bX|PVXr>f9wNK(@!4EAC?STnhFV?f{ zhoX@(q6hoashNxho?*(-Qpn(wYjJqLuJwE-LD#8rrnJ zz9!HQTpz%p6zylu&gq=5{U*g^-0kr`ee{i7N+BSlo=)AA7ewtwB05yNL~lhU-CAJ7 z;@12&)S6(c4Tl4r&pJ**QS|6eO>YLWm9;CA{zfjr@bMDYnvU?W(^-Tj76ZwpuM^`i;K(ZM6BBIJrX6p<`1Zxs=F?Q&5YC;D2d5L>I3;#W8n_OC;QAXAHh+ zGP>kVJ7Cq10AByfVf@RU`A0&1K$R_+g^@}#&iC)t*BUOe()-q~EW@e8;G4JeuTmJG z&$9Jgw*Cpdicg|6jrrblrcMpLe3x+9rC89V1tz4n&E@2mm|hJ+7swf`-edM+1BCnw zaf?;BX4>&^=l*WEu4QYi*C1GAAusPuutz9+7*K!hnoZU%D5ioG)}Hr${fHEM_+z?ps-y zGh}%3@gzrM`Y411CLm=LU7%-L+@5wopi!A)fR&}@yZ!f1DP}lWehSea338+p3LibX z^HA7cu`t*U%-Ko1Dw=0!a)9Jg`F*}JzK)agwMtQ{X&I8P;^C_0ZlR6xB;+I{x?)XB zDpcNQ+RXopySs3zvt9HwjXMPQ;O;Jg;7)LNcM0z9?(XjH?hsr91PSgC+-=R``}W!A z>^^gPx~sdVrlyL2fMV6Ehxh$G*DdM3%F%!iqhUL>r;TSHUA-P%QjEPq8*jA5RxPJ& z{s0bt<(4jDXDgX1e>a!a(2lGe$hIvi+rJD}JHfRwOiSppxa0nWl0^-C0PUVTNc2^p z<+h*Un0Kjms9ml{&XEkNJoXpSS|wEX5cBxE9W8T7n`_;uqoC{*%tUCG_Ey{|2TrIt z=&|U*U2+O)KpWTeiee|)0UO#wRvJbDgne6ggu>?~Saiq83-g_+XQ) zD2GO2IXeRLRR0h@vmkOi4HmaxLIwdMWZ@rd;DNQ;my(t9JxyPjQap&digHE*<03IJwD zW5XAAjwf4#a6<0z<)VK<^A*tluJ1+&@WmsPU?*w(%^d{MlxOczQZNF%<^X>Gh2-DT zOYq6RcDG#_M`t&E4?cB)KBXYM1$tl2gf7)k6QYb(n!VCUD!$S^5T)nV`R*8Q;T1 zXwcR!GW=a3XkJ=2ekbJj7SdDTgfIFsLP)Euu+?ho;w8}2^sV-}9SXoXRCjlvJ_QJs z3z0jy;QvIh*$e2&o{5P;9kPK$shx(tn~=Ngdw00q)%u=s;HU{Bs0Gtp&`*mwB$U>Z z=N{ybCnHTIGI}Bc_V*%2rkD{f-}3{8A;Ac*z|aB0FgZXNh8|d6`X>z2s`>O$UcxTi z4*L3)m)8sU^X}CKczt>&f+IWdG7P4KjY zncQzgnbCNbv5_VnkHhw7N*f-Dm@>f(nHR{;#yiRbdH8dZ z&$M5CQoyv^*`)8acZMui#i>UI%Yd!RQYPFU8VfH~$guCloLYiy!~;p&1{}eI`_8fy zMCjjlG3j|ck;M0mvW3PmJ65sa@~0JNkz+`TqfLi-J;FF9Yw?iO#4$aD)-)DMwfCc% zm|9~P^N1tyg$^N@t=0am!=_CLwwS~X75kaqFqM^o8xE%li#!#bqau164^ zad+H>wLNco&@mCB=_=gM=@=qBi*bj7#Ic0FgbtAFfk?3t(c>IQk5NEkTyNzw8DcZmswW<7sQKtVp(I9 zz6yc~qWHqi_`KqZB&)|!-svSzQ2jZ<6Ko|NIuL?SX3~TD3uhI z3&hJv>o8yXI+89dNl3&5wym7C;_3@8sO`tj;?$@sxl9+0tj!@}5VL&*5bwWp7}(#L zdxY}ZT&ab>>O5A)szUn(8dN^vK3{ZTuE$E@-hi&W z_&NQo%Cno4E9Fl$L{Yki&M?RBG|n;GZ?p z)FBfj4(1}AR}xE5C;i%3M!Tv6x9?D*iGML_fl~c*Ii+nArCpWCmea*JN;fGoyBDIFvITL2~k4rU94f7D*H>c=p~M$pK0ctVGB{)Rh_yCcWjscFod7vcI1D zv>R|E8Sf{>hrQz=#l>hTi_=jGVA$P5sCx%NF^?hHsl^WsaY^)GKLdUk?gX8EH+WHH z@(OzPg_s&LjA76#l@q@+Q%o^qFRLMaOH}e+JYws*uTb#Hd*YGcPo7P*)KFPBz2m|S zd}A&N65_A6-gZLiYF90RJF!WJ%;)oR3Pbv#sAPypQl(Zl0b}8+NNtaYr#j2p z`KjExr~*FzldTN|Rvqv?f+7?6yXoESjYOFa-R+Kf!h+Ya45k-1=uAS8(GelSTYh0P zX0eipByo#)S&ou)2nh9t_(3Ba>(V!~s!!KRHx6h?w!g^Kxx>;pv3^?mM#lqbm@%dx zsi6EV2}M#mOqzV?b?77r`wp1Jeq=gcv((Ed9y2VaM|)h43#AKa`*4ZDx}+tFNHC!r zlgM4Rq2Sjz-3IuS3-91rab;`4mBMXE8L)npD|i0HT2kh#X!@OMlnNa3F61mc)B$sZ zXDR0brWBu#@KPaB7u9KJeTx*C0S;F=33Kp9ZdQJK>h|bPMd+A-5NayX!9J?j=qS9h zW@xZ}Y9!-s@s7K@69+BbLbytjt!kx;cFrerJ`(UBpyV~eu4?Yh@}b}NJcRE=6T?l{ zDpJsCEQEgG{&Iw0k3qCXWI$%8m_m6QgbVTF1xL|5!^m*W1%qMc!R) zOuW9lXb%FkD8K-0M)o5P=U6K=|AB_tv$N9!oB|(=(agkyJ8S+miJ4P`sLC)O|9cWs z`1d5H0?GehCoy#Y_mdd1f0@J}Pq4}U_a-rA>amo`R;=Ho$~^R+2*-MM;5o7z@~}}J zbUbK4k@aXRhr)pIC7sz{)rOUjQ4N2`m%P%aS&4HyH<8}9jlOR!(R|0o*nRC433fOQ zx2LGe1ICw>M!Z1PE9!oXi{0xiAD2#&SC<)-hPdVZUP^u@6NCMv<7ss(lvqARxe`PB zHR!wB24U0gln8F$qvYgQvnOEHjPa&SpJHqOk;Tqv5!T|fe9aVoo4*-ady9uA0T-C% zxBTtRq^UXswMsaHBiX5SDlsd(;Ge|&OT+35d(~Ci=^i-Mub{L?KlEz49p+Dbcnlg))$7Nw$Nv+q?7U$7u zFfEUmxp3>zGLbijvW6YCY4g4EUhYrVoZOe%p6z$e^+T-wqs-Z6Oo~eM^goLkY*PJ# zfdegH6v<0)4+Z%RXoJ%+PEOQvQzCk|bHu#c=VZWW(fqK@xxGydh5 zQOlpEJsdTb`1?>psJJ|3v}+)<6mV7b2u9`Q zHU>V=kC1NnTuEo~s@vUeWw6BlZaPiI5P}~(UfZZYCtwga_i$MiuWw@a|K3X98q?O0 zS}#QlluK-}3h%=GotolvnDUsz#__xR+9($b5PVot=ENXU8XA+uD~d?;U=;c&Z6%S& zKk`dbANi#~V17x!()znlI1c?k^GiSe$}bgf!McRg-k^6{-3dkJd|B-TNqad)LmbSq ze?LxK)3Qn{pxXlWncm;2;g-P%Myt+hVxuTb^^h}63|;AZ*-LFUWdz^pUsl?+q;Ojq zSMl8wKUdy%j^(I8HC*6sK%Yj}^-rrHJ9c-omfD`@&`ggUXw)|$81+ZUt&6dyo@qX7 zmR9P~=GcHmr8+6FZrQ$FD+?b>Pdz%QVM1V>!4xps@N{}o;G|e?`zR_&&G&_SVpI>0 zU)w=6z(Z_ud=rQKL-R{?lDW+sW6zYd)!nsYX`8bj(jHk;M+R~~ZJjGBTG58+mIQ#@N~*3u2A1ssEK8BX!! ztvhpH%_cGh)TPVBwK4FUs|6R`e)E<^AT__+So>$ctsTCUZUiDh*}gVP_~zgIBKg2! z_VJzomIl=8f#}-Nq4}RL0J*%?y#QWpUV@EYKv(jo*}@i?s>|$0+Ny4dKZXCM!}M0P z)%`<<3Hc8?%=v$T!<_xA4s-V(b(o|7X&vV5Uv-$*|0NwpL0o*$z-W!{ztmw4 z0RLHs!E5+$beNS79i{+aq`>rlREN3$-=M>6{3jje@-H1m`@hv;mj7LcS!LDxdYF|4oO%1U%hdZSVf)Sq%MuXBKl9@n6hhnk2~^kZQ61hh{OW|9_ap(4GBv zXEBWbBeR%}|1gVb{ogx_c_R47EavyW%wiV)*Jm*u|9KW8bQJtQzluTl$0}xZ)2M|# z_od~9)oDht2^UHA{H_H>qkMD;nKXb+3k>N=l|0N+Hqcg~V5vvCpfwuuf__(6M2-|PwWMAop+b#g&9glyUn)V2;&j2X_xTxqXDTjTD?3@cFi2ylGLx-xR)#lYg~1BBY=f@D74u`VN?xpPW0pA%zjOZeApi%B@+XqpTG+al4Q9Qz(d zn#xfWVzt~&G5~cgbg|wPQGvAp;2W(5_TC-4O-u7N<(m6TDP3~iWxSvGS~{%OH0^3# z5vSla+5rdmrc8^!+W<=fVGdT~ll(;$3{|JNU4U13ua;eaD1k-n`=2PlSGU(k`>*e= zZN7hAco*x#3u?FP^ZI(1qoSvMXG2Qo*_@v;9Y$2GsH*B2a4o2+zgX5~ z2H^0(u6o0)yF{OAn@RLjUmtFd1;4(32k5?cX3u1g`qI4CqRh2^P3aDn=tjzs(FD6= zno2Z1#uJ}vmg@}8YrpTWWOf`nX0oroyj%U+t%c;D2r-KTV)hpe10!SxY%gg*=i!Oe zXcW)#TW)DC;`yT0vKADRT25Ivxld}AuzLTdGn|rA%KU-#^Q9U^XSUeuH9xHdvuQ#5 z*g2ttC9sCc%F{C}JKm}h)qs-qRU@PPg+wlrO=1_t9e^qv%%nD3pHWv$+v+ggoci1L z`ghDLrA9n{5d)0NnD1a|^8+FDVE;nt!}X?(m+k#~VMa*J6GBmZ$utd8O4A{<#k6AB z<)7uJC@HFTsLo@FACDNnWTi(&Ro|7C&dGY;mcHi1@6C~fzJJa3li;R_ECPvy-%Jf3 zCHrT4$wJ9)@!#7^1>eL!IYrOU!T;4>`i4A%C%aD7o@Z~o2cF#aCtk+$(NZY=WLW69 zV=Tf9x!FtVIc7!0T~2QR%~a>3y@d6T_7e9XJTb7n)G7hUE|q+-^nkjbZ1VO1BBp=0 zmp+euZy^-s9I#Y|_uEx;uw-H?IeC8TB3PH_&;RkYa@vr2vfjtC}}6&uJu}jQ>e=o)J1@R=|!IX4Hk?>6OKi;0ev+ocB7i{n0f; z+-$lK;NUWG5u-O8)R{#{O& zu>1L{ng7#rw$a_wCO4CYt=F@{|5zS*S@ZH!YWj@nXVH`3Oxll@WQ=j385Q%efK-S8f@YYGvpj zr@vjZIH^J$uJe29lKZ3z7#&8Q%~%=H5Nd0=^m_S}KTSGw7eZPMKvg1|7w82}swWBi zZ3xSvY9MOUp}x(_r#7XrUv4FmUiTL)X{7qEI`S_}eN};IQoG5GOQj^+2^!&5ahYjg zXm1h72p0^BD~gJ}q==u>r*J8zW1n`;w$^!FbeZ9Jd_jVr5IqKC18Qqc&G<1}#iC@*zTxuLT_ew3F? z-qZDZ67d>IV19UamxWce;t~+aVB_VXNOgh)drvEJiIF1@zLf%JF^_{# z=m8G}{F;OF%#{QDm^>PGcW$ZRRCMTSX~fHaS<5;3?x!z z5wgTACgIwKBMx=?qf#A?9x)j7IM7xP>g16Gu5@ZyX-*2*pK31m=_otnXS_7#vZFG` z^hUn^aQ}{6!xffrvG*{{*@#-Xhp3Q)cm^-r+i%tX;g^GEL>x<7W&om09a+yM8+b_~s0!ean+$fNL{eJa(fP67Q?Wo|pKB za+0`)r|Rer>8}VSsr|w(lHB=OrGgMi;jW?$=n>{Pr$#I3RK!Xi#Iw}r>ix)uVs`#q zujY#$-la%)NpuUrBO5F_dz$4f3gz<=p!oHhd0eF5vEdOF5!op^&jQH73Jn9eWiZ)d zQpv(_WPQ#9=;SyB6-x$*;k1h7dxr_k=WYmodvDW>X|i)vV^kM@7>^F+%QE>p_tU>=M1D z#zC-XQKUb`a+b1@{iYzfL#y%>v{!KdG}Aw5e~)qDIUTs#SK12FL4Bc7fS7J@)3CAA zoFyxAAHB+Oqc(%e7}lx=u2rgb)KO7fB!HJnNRr5mg7wp{#VV!S96o~~C+RJk8aCU5 z^ilQokiIK!z#T^5SHg~BxDUI_l3y z<(8Ve3gac%KnQ56-wkoV0TC=2H*?7Lo@$|f`dJY=xPs}AUrM_1Zjf>vMMqJoLNF05 zMj8pvTItefopY@t-!_vIqWsJOTgNxd4CM+p+Mvb0AktA?kK^DTq9ZjtDP`}Hs-HE( zPt+VHmV}$Xs#+oJh7B@KpJ3j`zmv&QzHwum6#!@uF)!;R1lCJ6_GSjNEt&FVd^O5z zupQb~b*XDIp5cNUwOQEhnUg!*gur%ecNfe0=1u*Ydp|uo9p3Jxljmw29L1ABNz!QCzj$xPn?d9P@A6u>;t1xP2pkgnNyzQ5|?qA zsk?x=L+>Ql&l8Ts`1F1#dvvfkZeGQ%@I|N8IITgd(Sq(Kkooo*S^h%MoOm#*o;TFqevoG`K=T+TIHe4x%Hy-tk;kAZ9r$5yZuYg&-Ch zN7tZJTbdH3#Ymc{DfV%DJRPLs$rRU|txvpIY*+9S=^@;4!`)ms(Um~J%3Gw-A zYlRPl(5>c}>oIV|n`Z}PzbtYFAvLl!HXM1LO<+BFA?bprHBVo$JYEkUV)@fNoCt$0 z9=t%{(45Ce>#{NAr?`m_>~=ISrgLiz!^x%V#~I@;u+}$AkhE!;l-LQ@atl`m7RHV}EmX!~(AY(@DT5WQa=G-NY#w3My3D08&$ZGt2;%;8lIi+@u9{ISr>y)p z2-!+G^KQUydyO1SyQOp&Rb^}gPQAPg;~xQS`+*8k1NIJ%a;#3IXUwj4;&@bcFRzuTV(R_Z`dK`N4!$yEao<<*o@Le>hb^y8K#Mzjyhm#JSn5eHrm^WdyN zcRGbIfUPu{# z;rLkVts@hH6*J2F1emR4>OCcOkjLqb7U0lO383u zJo1+XIzyp~*wlGLOr~jzo}`r1Et2fqg~ZR}+sIU#A(N=( zkT$|Ly`Ur_FC1b-vMpItt%g;qX97g0-&vuMumb6+X-8K%{4hC8DnSv}rF>O>6b1E| zQYLz(emCtrMnWun2_~bx?P(B z20^q+0Hoh9p*_C#{LOpfg$$5!?heMZ6(}M-R{cm)Kk9yToPKjuYP?H+(4-Sf29yctn7o!*Ta8kMxU7%{jKsTH<*DDvJ4SUFYNNhzH<2He; z>}9ZAq&#V3yNc@3!IFfo6{mfGLcv-Q${ciC)buXi649R!Kh>#b9d?1}@DTq3Lq6cw zy6$ip(ryq273TlbtHbC9u!Fw7oxvY5sfzG>*X`1XO3Ma`s4nc+LF8qMF@ z^G}EsgPZw$_+VbJx%zrPB+;QnAy~14$mxI8$l)JmDMq_3>YpR(>pB~K&H@f?z#Tw{ zkUhTQc*Fz5L3;UpFt-4a(jA)+fLsu#Qu_JAx5nEy*9(i!0Ko0l+vb}W(a*3jmqTP! zD3ds2EL3((v)`~`%b@neX<^MUeKs}5`mujL8cw}{e``2Fzr5zIEmh#)A%}fTYf^v> zr-_e-6Pdu6?lcU*-v;k=WzgL2KH{U{boN)nsqdc+r@N1a)9D|c_>oP7*=V>=1O?-P zp%JOR$BEFgtz5uqjQ}*CG{<+I0H3uK=W%ygXg9um;-3nRi#HE%8-+#hl-lu`HgL9grAMesHkOu$wi7OyWR0YUWf zjgQ(x^IgkD_aA=8ET}J7+rM_A=+RKRIf3_O31}u^zG!n3yE(ujWrtIhfbVqNmHlb% zm|l9@VlNsJ8vGkYdHFUBQ%bn>-NuMG_X+qI zX2pK{)vqY08AcuN6>L>pTs*w$N=KshpSdeAKSXHIc*Sdg#xXb2M~{4>#3{58T~L5G z6X5d->bd=i6gI&=`POr4J*I5SB!YsY6<`frw7KTn!svtioJ1!+w4>(Bai92}f9z9(&LqVPkZ7s z+M=PfzTonni7J5lhM(CFuc96YRTPmNd2?2c%`qwB9$Fz5+kKCG6^7R2QMnE}9*n&0|MG<=4D) zVM#+ml6TqMfoObI<`_Z9qP3?YMjNB?gpCL1ZzLQ{7#IDPDWjpio`FlkPZ0Ey@gUFu z<~hJ|T6b)@>k3W_jq>I^;>($RqU{vG1(McrL@8|*bC1D1R>-ZVG7&#EUGak_!6OhZ3x(>vWz6*pzje5<&MY3VLafr~vERbi(1Lx^TuznS%6T!; zr67NTP(>>ewXrAGMs@8apJbW_9Ny!x$L}Eez~l=W)&F45!*6684{KAd2M>yXLt$Vt zBB$SFGU21O!Tf>COXSjZun#2vErJl@APx2Pssx?Qg(deuL**?*_xpQF4`vBNg;V-DtC{F@}m9Dc|!|Qlczq}xF$OXkW`Hx<_#eZw=Ws}HC^L>3i;cgLu zK|b-e=Bs3?H>EN3RyU41mW*>9JP5#uzVw0b(fm`(+#ocQ;YX~D9HLDUj%UD~2>Nx4 z7e*q@xKY9ufnfmN197Ge-UBi^*EIft@5W~i!W-o}QF7BquVD$5iFX)p9FLwC;b zf>wa=$`C%ao2UbBK2BQ2YeR1MR06@vNVtnVFT;LZilLpDC~7Bzleq;hK^~lv95r$CWdWY;LD4#$@NO2zT;dgZs7+fN3CG^Q$vKVI1rorLE}gD6 zgq1iy|Cun@rBSnb$Iw^c3iOy8NqZ+{{kMrWeb*qTVfXf(C_Nf^iwOgDS0Dey*9-2T zDzbhN=12tLBOHDov+|guKu(2)!L9;qEn(Z^N%cR5(kb~9fIJ6bIT4v3imf2`wshb@wHSRG>W!=Lm!I%8f{HL%;8kP0O5WcmrmX9z)v^F104 zFG)xVhe(vkdUZsWX`?Xy(I0}LQG&RYghP>ZS=_J-^S~W*!+ts4E#lG)Ka!j@RocpXj zyvTLuXian(7)JqXOR4kKKh|*OSS~A7@uHu=8jKBNgM-f+YX;=fg*D>_WjU@y+N)`w zB%{y?XdaffiZ%bLE!8Tcebkobr#TZX1a>Jm9+EJzfVCxtorH{_(|breqd6oC8~Lay z>$ZNg#P$}izcw{Oz)cPMaNSmeVsdak4ZBexcCig@k?d2s)Cdr_7d@cSmCplVW=~mx zf4Uxas@2?W#eUM-Z#ERTHPj6zIUcpMN^CAHcR!3}&GK!u{9wKRoY1g%IE zYr=hN7!IDzM05A?!`J*q06MQCIgoF)&c4kptBRfHGD4B zJX=87dXI#B;PZ?wq9BO#0(54oets)QEgV}(B^!0*`*3D{ zx6DxJz0=>cJGB&X*b+9aTz_FzEq6`ju4FULW#|2*YGn|eUdeRe;%MgbJL_wDRHGWlIE$FaRrz_LG_2+HsOvTWQtNj=WnDJqhJVqPVPDf&h z6uNDn1f7Ut|Ie;8t(cXggP5a8bdN2jDaNxk2|5%A{i2anobc^ajSI~LJrhucc*(mO z^(JeNlRCaHZZWD~tW19MI2(gf+BmT-GWku61Q!2x>E4?z(ips0Q8O$)Dp*u!JW9Fo znzFDM`KeXN13pre!Vs~i%GZ(VhB|l(CetRz9xmJ=9DjfYsjG@ zokWz}O}enXPFfE2(+V60QZrIUu@LUBsU(^E)&hLiM%Pt;VpWQu0elJa%i4QK0YYj` z*!7@(7W~#T*wsHFmZ5tdGzL#JmLUV!G!3`ggWDm*a!ku1+FrfxQ(_mA?Ok+?>ISk+yTvWa{Y`{%F0;tob?W14* zaYxY`hV5pGPlZBO&8K^E{Vpq45iGmeHLr)>c$@Y%X->=((HMj@dXEFR=gR)RTu8*` z)4OX=NyzroRPDCC$l;Gn2(oi@Dr|4H>YEsWSH@vz^S%TTqbHcyAQ8%Q=vXSDSrx>F z^=>Q3OtPE@FH7_S=y7=|@50uNA3mR_%jG1o@#JOHio7EvVnOb_u}_K#u((95=dr;Y z(rP|Qy7o9=VRS=%FhIJdEP}Y1qBmN1@CuY>Bnt_L)UMN=7-Tvawvx zom?I~5=e&lbMgCD1_rP`NnC|KT*%Rn!q5VT#D=8woesEqnzw+d-+)ykF#1Tq!s&zM zL+@arnRx6&@5>|akRCX)8yhZVp}&guO3mlf zsNUPzNpmZARhguPVCWH)@|#7I$4_gPUP>xVj+;g}+P0N*jj(s?!R+viEb@O_yIj4s zDPD*CT&qFWa;KUXvVv^vqG_jy@*r+D?I#~vMVEdLR}A^m)N$uSh*kBfr7i2zM^m7G z2X9|M;eJPdwML3E6+-?`Hp zu9)$Yr+#85SESn}N_RFKZHQXTfVwE0yV@GYh$z~y~m#*DI7 zk8cI%hXq**$Lu;eL7w>t1Xvzgt0Z|lag;PptR5Z(TUc0Xg3pg5@wrY}&SpV|i(&jz z&0UMjvTDY{#)O6hH$d4pT$ktI*JC+Df}T%?R~u-f@<3`P!Yf94zLi5`SIsk$g?_UE zDzi#Mz(l{ev3^vx0Y8BED9IBU{y?&ua-;5Q|7Ewax69{y%?eYivgJ=APR=QmkVFg2 z_)@uC2D(kDj&aMdq~%{oL6sM0PRo@)Cm0Jmy4`qs1EGH4HAeFC>yF=$}^Hi>WB$rrB9m>}vI%*Q{fXDH6U~*|APL!|$CT}r(A@U@qDZEXWWB!E1%K1b# zhI8yIhQ*|mv5g8_xb`WgLT1uhvqifh8@Ifnw>|tPT7Q5A26cID~jx8I-3`sbM+zMi^r<(^AVsscIMA(?*UE-=;#(l@6qD1cb_j_?61@( zH%nFWZOl_9zUdqe8K}RE#k6DdP zT_Ri)akVf!D{Dk5c8VuD;m=nF6nRD@Lt_6au*@Q zc7~vIkbN5Czd2tKCDq}4tT45Nj&Om&CCq1W$QqUIE7z+pq*^I?OOryUMoSJD$EUVo z?57!;gnII|tzaGcEBm{j$eWT2vN5i8DOJ^Mt7xNEH<0RC4Q*7BT}TFUi_>@+BdR6` z=y2j5C}A)L)CEf$23sm7X}Hx0Tsp^G!Jl*EgbN@eN!q>vP6-z`eDQCQV(tm=fM4?K zf8NB|5p-^FBuKwyt9<@xHS?)4z|`;Z&(eMr0#$oJn+9sk6>V5{N?3gLwfN!3sHUas zE9q#V=%+VN4zOg0yu>2Eu*i*50b`+G>5yF|&z21bUFB^we@Qa~ZPEOA@g;?nG~&lw z2Ev}^Y?R32b= zdVRF}EgO0(Aif^3wu?4es4@0I&CGsKGYhXBD%l^@%v@NTv08%#er^v+DLkvxh!IHv zMue>E_*qv_*DlH99k6-G-a#Y!NNm|eY1Fjxj!K-s|0kaE)^U=_xY4{(Xb$O31;amI zg=(%eSSr=s;T}$jOz3*#c7U=J87(_(N?g`iaW&UiQL~J#V7E$szXEp2E3Kp0$TM%& zDK^*h(OTgBkS^7%uX@k-qHZ|}5E}cQ>j?}lJ@v-u`A&5CHIF+2Lj6!as2L94cWfU_ zO?+s3q4fte8aBOTFuHN7z-fwpjDwKxY{D^en^d$Fs7)-QcgIiVqi$96%5W%|SQ-)* zL&c3+7+LV|Z?iYO=kE`cZ0RFd7OJz!FX}D8L+a06-*dt{B*F)Pw^z%kg?vg=-=qsy zy}2^>)D8q{!-1e1j<);(4~YfiAem=W9jpxMzys&1^UkedEIJ$P_+tc^#OfoT1gTkM z(Jffx@%bLINVr&7^m}Sdtb1`Ix}`&cge);L;oWDz;#27j3vF4Ws4S5U+|%h8clb^K z_q(XJ%XY6s&u;9yg41^Ib>x_6DnR^wtY*H()*L-Zdk2(gZRKvo`TXp;aBF_$Zh{)% zsyc)MV#X@=v8uu6XKd(QsB?PdDiFkq#9G<>42THU5b6BX)2{Yd)d`r< zCG7oJ)kH5H8C|_^@}Ci&EkF2PY+b$frp~C6*0%2APh{m?d%t8RZLc94ErxU5=D?3WuB2B#9TrfuMMYYb>-22I8?6 zRYZcvMM(BsOw4G(ZE@L8`C~6|U(@U})J9HcJI(nRSzdMqASf#+ z%mNB(TPlH#Zy-|yHwcZ7_RAudl6!;7wbiD*&jF_jYLVO1H{ponqzwphNo^<=Xs%kG z=>)RQk+UPV{hbs<{MgroDG90SD=EJs0n^h1cpEScjrIi20z*`y88;4E4SWU3k5x_n zdvNXM7@`G^nDC;@WaTuB!QPE3_UK$jxY@jjG_s z2>^>rFd8HwX$xR<*d&SPY;8axHOar(lPXa3Qz}C0MVhrL?q9naQl5G_@A$cFgv@@u zkk7zf4V`umO7J>$*{_9oOm}}&52GHN;rSsT`18p-o|#eV69u=yl(rwpArm{b_EaKRr}$9S9eF*vY~?3O7>p2n_f+6ktSAuXNo zk`n%Zb^{v1S-uamyT@k9^Ekv#V$;tGiSZw@(b+HVHDQ|#qnpG*b0VZp$Xa|u}c z)R}knD3yAAJ6QQ<|2=04&bhp$TG5>r%Hh##a2UsAFiwxeL`k?oxYm4}dz>c?>X=R< zfg2VLw@BMSKJe_)89g;^geU(R3;_Q9TOA74c-bg}1)RTo8BV$I{DkQ?L^@9nMw}?t zY9A?@F`85VFbDn;Pr#Nk-H&Zu8yL@^l>E&r?o#yHpy#1cNP1XWv+d#rJP;F4GVzQ- zI+M6)FQMe7?KM=rSlvLU#7RFFT2f5V!bkT^Gip5_B3#VseHP)$8v+MWYdIk77>4z13Q<0@G z#e)Wmf;iK9WjwC7vmGrz^IMNoO9~c60FpC^?r+>_2e9Ov(}-7(ps_nYP(_adZw?LY;aO3gv`TQwggVkrW${+be;q6NNzYr1kWA6-3r3RVkgvLL zYr#;gAsU&S6}Lwe64jR7K^JN+o7`o_n&&4vy|%*)-doGaG=*gK|ic8=a6QbJBty zTRY~4AL@|&sNFnZD|j@Mz;X~8&H)e0z|wOuPnSRw^Y~F+8kmHXKn-R3s4fu?VnK&8 zp6nvmBR{F5Q?wN9j-pDfjE$TG$pgt5n#eCeat4L*gPch}#sBm{&bVNVJJs?*e2_E9 z^7gJ}CUf#i^p9aIBhFTZtVS)@C^>d zya5h5N-{m!*0Tu2UKSiR8b6KLI>K{PiYpYMJ z2Jq=!NbCl-FP1I;_L)$6QpL)!pk`GEfhoj}rp(+82%rScZ0$a}OSEUm;YMG84}NrA z6YYcsC&(wXgYpMYOYezbqjf8J4B{O8LCVyK)waGxMJp{Je{`1yrJ5`@%qNql>0+su zPZHbf(t?xf^U@-Z+>wV8HV2iiHG{emrw0Dmv2iA=>EV+$ub4ZT@ zX=OZ=3qy`G6B?q1Pco;db0y11eEuuEbU5nhltY3lTVQNt(ozn2edyRIb{(IW_n5XF zg23-p#ax@tIoU*lvln>AaVF1$bp+0og3kC=npBhQukcc-hB7d`#3*o_1vY{()UqO4 zYBNA@F_A>?&GA{m&O5+6Mag)U7xouH_~JNt6bS*7X>-V8<>~-jX2qC_Qs3=rDCt7v z*mjwd5puk|`UPrIzzGzYs%3^KP@N%d@Df&JtjCK}3EVCU43hF4aIxVvlb+4>P z4_P^2gm5r`;8&EzUGO8AHVRJ;tF};hC+V{yrv1N2b(bgOwSN3K* zbQ;H4WRlji=Td-|6cg9GY8*D9)kZG)i&6* zq#vIu*?Fc{tF~o0euY}xzK|zG0__QstE=55g9{`Hz(E*Dz_ebbjB%7L7wfWQdVpf8 z?WjkRh5T{}$_;`H)18CRJqjT8+a%Tjx$uoaimiAAywA-w zzLUNK`1pWPB9Tr<7}!7-i3a#^+{ngrh@n%zHso(Znh;wYYAkg>?p+xrfk(^phc1Gr zlv^M)^FCu^wHSp6GqlL+gCU&&nA`_KGtj7c#*Ah`fWDUKyE6keE>BSdAT;Ch7c`?U z8F%mfP{2)w(`me~CcbtD!H{YhcJUwC+;_jU}tR0|(9#y$@`mprrevtaM9 z23hWzV~|*~Chsqn9v<&Kn+pT(!EfL3J6pAkA|HGK8UT&AH@)w@50CGMx_nx2kAQ1*@}2cQ8GtrZjTdqoj#BAw`WSr{@MGvGAark&Yha@0}c!)^ga-rpvPHL5_>lAeRcV5Q#%X zLdyG#q~Cpd)92o>12^P|3_c*p;QmF+%Zsyg@>zRrePO%f^~dj+6TyXi&@tp+OBl9) zmM~Pc3EVntM3f!iVY4D=Q=pDpp3B!-y5<*O_5`!3O-}+G$mSP_ZpiuXzBfekM?Kr3 z`D1RHlI#8J#P|gEOw!6f7zx*pN;icaI@r%4qJdu$YC9PUObB%!Kj0mTmR2&f+FVp* zhx)YAUeqM4KYWI*CF?KF2`pI}iX1J@Z})p7k0oWl1axW*FQ0YIrAzMvi6Y0xQBAlA zKexY{eC~ocPj@lXx6hMDA-NFecX22N8IRTeBq5qW)>T%kET+-ovaO0hZ6@MEgN|^r zo)*D)Yls6_+q@u9>-s~3-|(C0+#e4*Qb|8!(1Y8|1yN)&01=>@GBGpplvb43?s5(Z z>P8YqjN;$rtaBI-w?$E+(>?#dEHI^K$AQjwqO{joS?Z44YPS!I`fHzFUIf`yf!_CW z#wL=sK3N;+m5}uw|3Tedi%xmNv@LNp0U7w`@*{tQ!#kgj2M!pdo#e?;c%mf+(KstUNBv*Q{$^Fx?m#wuQ%eg* zR6PZ?#-BP*YfMG^6To{UmsHYtI*8|l`S8|MBZ5&)M?5qX3%CsfT_*oNuA>!nE%Yq( zl>vspVP|XQH!4vxIlabid#6)3nEQ`-`HKpI!N&{H4chUaT2Krs9l zvq!Aj;`MH?32BbG_o#o6<1m)iYXpo}ZwDKu_p@23gXj+54ZiP5e@}<+!oK4`jY8u% zU{rr-gWZ)^R7f=sr`=PDfFhA`g?ShKOO>yEZXc7xJpT5LF#bnJcpoz&%oX{Q&A9## z@hzyMR}VK*RC7_E$qpkEwh3ejHAie&fDcY8)2K(GzOk4bc-gD&I8jmFK~KD=dN) z8q&(1nF2C{K_=G(4RtU4OSaCU;Tb*O-n}7YN)K5V!g9QXn(wA`-}MJ1UjtDtjzSNo zDOuVkGO~!{e42V0$c9o~?wzOXHwC@or!6dP2+7g_XVi$Xo6j?dOfRa|={V0*cvezh zXd@kR`mpGB#trUi^sYd^jOmhTn*AtA%d^-xp|166HOB`2{1^YnA5ip;(%09_wW!;B zxDCCxZ-uE5sDGH#ko9)+fMeJnLTZTyh3TT9M33PFeN8IjE(2LWTt-JRx`wFD#jN;O z1KHH?VvJ+aw-;!b*H0g{TflkzTpl^=gyp~ov;NqB+QFazo_5cSzskH`Z6-K55D(9g zG5!C-d!~{qtOy9J<&{*DD{p!abzaxt#5}XTWIm6U)PfKsbYxH z*91{6iYecQ-9JDHP?qOja&?b~ldreDY=Fl{GV)1|+8-3K`ZPKS6=@Q0pq03=HLj9D{h61OgR53y;m!o40lUr-z zxsS@IvCUI^(KDjl1p~l_mL<#>{w=o@vtxL8le`uQx)9}o9$8E>R3p0@I}6M$l{PCi zfLOo-b4yt9)#VvvDp2KeI15sbQ;?WK4D8d`jRs9wa$~=+SgDP$g>SpAOpiV$GC0m* zPF6Uk0imy#)IYqxcdQPpq>vWnXeEeIGg2ySh>2H5RgnLaTk0H~hBtjGlD8b$VU%ys zlXr0rI0Hczlj44IHa{|r6H#q~3<7=IigcNj$;)MI_pNpyHj?beClBw;J(=2ghg?FnHimK7WHIIzy;&0qrBGTi^zGH37!C3}yy zbsBC+7Z{njIe*n?O=b80*iO~Z(N~*BFH$rAY44Jme;YR7-jJB1tY#80Mbsm{mQ0=B zr7*^_jml^H^rkzbJjc{Qzm2F$3)177`H?Hc0({41vgGp*@^_iMoq36wR8IdIS?$ZG@{by^ zU=+>)o~DcA%ZWoBm&`P0X%>vymtk8p=ZV;o+Lo@jhW&MJf$ricj&H}SNb>8+ z+{@7nZX!Yw^syj5mRajLa76VcFwaSsO0)=2Cf+Sr;%gPc+>nGK=o zwM05J$8B?SYM*X(*|E0Df6PVF{TVgCe)}`_(6p#YbFp1P0>^1tR8$cfpv{g?d)0bD z=0-k)<2%!fN+XI{GwLbWmVII^C(Tm2fwP6pNZ#V2NZNM4&{O=-rJw;ALO|Ks~5{5lM)i&IhaW@bkH; zX!L!~{PW)i*xxP`*U2$n!d_Av;*s|31=`w!BCni~}f{G-V@9*{g72Sjl>{f2VuMlyTUF?$^2Ju$r{*O>J*3~Neu4~BY!#P{b(ep09idp&; z8}A|aNK2a$HZ1-)S1ZemJt$1q=n1IDuYlxswr~lCqx2Na#)b}!vin#iS{-g$2CkBw z*D6m!tu*4P{qd&zvF8SXf;9N%UU)vOnPm=FosBdoCS-*oJHBuDZcQJZiKcm?!$?t{ zIjJL~M)LiY063}02J8LFuwTVlfJuJ82r@{WOa{1~Ig@OeKg&{oi!w^ZZ_2}_0~d2l zod;d#JJ`W5d${sM1NN5Oc&`tEy`{@YzBHBEo(bf_+B@dGww21A@gVt!kNVahFo#ES z{T5iHLIB3s`-A9|{w&)MjNOiVU~j3^k(==6Q?faZH-7w(`5a;{S8z0TE)%X*ZNA{6 za_010BYbWn;i2>#&!_`NjE4!XKBuh>--wR8NmgX}!k>FM?A1&^P==qDk3la|d`*ax z6I8|YVe3oV<8H}fVth(?+c-NU6O70wC-$xi7UTp}U$^Hl+@@K-y8Q;*4;8VK&S5dMCLb}L#t;PTuQ?Q)t=xnzW8yd~)iFe7wuA^yb1bx* z2{&u;Qqo6{UE<$$-_8SNUlQFhfq1@-+IKb`vRavi5j{!>{}8*o z$4H>JlQp{HD=3$|ElbnY3e_D&oqu+G!Ua4!LzimRxJjN!S9+u^>!Ck%(pKR?;xnjO z7MO0f)LD3%+PNkN8s*{M{}Wu&H&JF8hZuNK8vG}?RFquLOldwX?mU`P47NBbDwtCb z+5n@J@aWv|8#zX*W5vb;g&0e)&@5=t+%d6)>{BPx8`OEGi}q;kZPKvX7hD_h?Af!2 zJCEI^EX;6V=3Ts3QN#JTWb+8Uhmsy1mFOq>Y=tS?wo%vF$E%p;Lx+7AzwpX(rD|WB zLoLV5xRh9JC=QC*>ks#F_DVq&V;ov_GOMGsGcgM+9g@r_$e=@vCdU@9A2x7snhD>YEqyrlN6fsFIy%(>?TN~`x%@$ zS#~ajIPW0w5qP2Uva($$Gu2u>KyHnpfUR+6L)(}{8%Bu4eS|X{=%I8I!C~Y5`t@?j zP`8t1%|1<+g2OgGBkLTzXOby;c(zmqH`SI{+qgk=#Mb(4@0^Ec>xug5eyZG#VSDx_ zf8cKPl6WpI&8bIoLCKoxM7_dLwW0ETw({3<%t=dhgw8p8qTPi_je4_&9S0zn!(<_h z8q5Ot61uRH#Fn@=EvC71Tq~zD=|I*=UKFPZRZ|;XmUyjGbea637)zP3eGlk7lyhs|0I`k9L`W#%Q(Vn z@qx)DFyUftWsxtUm5ag3+JUh-{(+NUU;4=rRR#Y(gYnUN2H^8~{r&%W2E%jeM`I4I zQwqSt!&`pr3n~)-yz%8jDg;Qr=zm*i<4WiOE#5}Wkk#_{#5$IYcx+L0zS^<_VO`sH zM`sbwCL^zf=+)~ga~nc#H-q)yDV|ok?%lJG*?f$m)cy9>3h6Xc8S zo=R5E)-?Z?-l8M=)`@F0(^pN z1A550c=$hf&;6e7I{v)JSz&b^v&4&~b60VfujLY$LCk>c{i&J4##V5GaBKpe3R6es zp+h0jK8l;&lJtGI_3#pl7mvb!d$s|mJ`$ji1VzE@Sqm5su4jRBmwj3>nek9|RQ;DN z6XJ;L$4)FOF0T^1I3NwQWiaZZ8FD$I+Z{d8NgGg*RBl^^S8MjB5Jk%|r=-+HF?fT@ zN0gP!r(>hNJi+ACRqd`;`Sg6hdDv!pjENvIbZA{C<}kmzrIuV3ondXV4E?QW=svVn zc?`#hrG@cr!e-ldyOXI>O^AnaS6?7e^FOCxqOl}XK7@VwNLhhT@-NLLY9ghDcB*_? zVoIHwoE$qi8Q`}*^CQIgX9m*-oWU%>s`Zn->G!$)b_aO5KVkvSfY0;x<2xxQxAUdBQj|vOdT!5JCAn@iy6q3pH0hOxYJP zlKn*8iBd=JH?eH`X-e$4h;TJZ?k%dO2h{u6sNFT~L%_;uCE8wC`9f~=g4w%tu*6rh zc<%fh(@^|O;S8)O&!f63c&J)|NFD~Y?(xJrjZ6ac1nv@CUbc{o3-K>vt`$j3zeGvP zg2nLICZUuna$s=jJT>+zKxpZw@AaND&udGr6L6W2 z4`^ER^L%Uv+#CQGDnX6K5N54InqozR78n%q@oY$gQ@er8J1CM~*yQWR!N8*=$i00F zwcpDyy^8ZC6-L$31Qti11#!5vNlP%(W!)&k2&GtsY##euyr*e$N)(ckM4VtFUdw^H z%q)LSnxbItRx!d^c}-~BOE~V9?$PFt`H88qAx?=`N4lL|10H}t8fwF zAu1Bc`fTl7t+V+W0Yuttt-joAUlVb!!!N+pIeIO>X!rWw$GSHkQEgP!TC3BiE52UQ$fpTqx~bDUDuMs zi`A@i{|?|)q-0j}fW;7@YMmfCD?Lbl+l&NtW59!8p+3%iVrmj3fwz+kpGK|gs^RWc zM^@E-t_;{l1+cj^fgOPjgPJPYRW>VKOB-)yfLu!|c|g)FSuDu8*How+!;(UwP1g4p z@5J0!nM^P;ZH~2AL-I`4A$i!8(Aoc?^MF>e(Rw;oPMO|rz5i~UVLiHd8K(|h0@kF} zu6Le1J|%_5gM^wK^gi0WDRBfPrFdv48}D5ImKUNgRu-d5{3Kg|&Il+BDR)SCJeg5A7hQiK5%ZZ`ko2^LB(_FZ z7_-Ks!-T_Y?1Wj#zrst*jo;swF|oHM>ajmee>-MBj}9K;Dvsv_M*hUJOu(cp;5h!H zMw^2)D)xPSXUfB^Kt)HRNH~~&k7srj<5iPlXinx!|LOD79B|HgVyck*al&c)(Jjsg zJO-Aaj^jj~i)m3?KW#T&)@8b(2{hu6&-Ln$jMShkKH;}GGH zisb?v(%s>mq$l|67nw>FV+9^0L-uEQd8 zTxp;=eLC;<_1p&-dyT!k?wyk2{=MbeOJYx*CZ6DxdNJm_0w0<-Ao~lC5tT7DgsEDF zmu8WxFWjm+&eUu_IXZ}DR?c_SDIi`U@Z0b)RMhANk89XbGk z!mv3Gooq#R_}3UEb~aL;$H>y`qV6Ee_}M_xj3mYa!AW#TwsAy#amNVD?(ftLxBR!G z;JA(j(UF&lrO4@FnF-vaJvJC}@f`=6pqefP2)xXK?1Gp&HbaIzB+zmrPCu?!LhHpX zu;1|VoTcuh%H^YV<`e6kF!ZRqKowfWor<}pl_1y0DO4nJd+F|;PT3?lCg3gFn2k^v zBlFrkQtywT3Jn*u=vkV%Z-~@~Atg{7e$h_Bk%$yZL`(OcQV24ycIlt%y9-7Oo+*9_ zOQPo9O}gWSzI&r`kgALDNiGLF@nEjQ;A9A5g^di~V{#<+3}uTd%T&t~s4+@M zMKP1WI6fgRCM_bv%-9#HvTir}k&V2pe`I3R>PXlx(ce1)NCYA?q1ls&;|Lz2ni|7E zWM(sdEicG2SHbCE6FJT={H0I*6Al(WWt54O13cC?SN@n2%ATomjL}mus#N>A@WkZ#l{Tv>R z!iB94dDBEH2>MY;5czrp?Z@X`!va_MBdUerG(V+e)~%{rjax$J5qCbk>+E*<7V*f| z%Xtvm+fRY-x3GC>_tD&fc>k6ysG;jvboJ48*J63LXX_QzW2@!*rUi z6HmZtBLvMt;mVqfzJ-}=3P1Z(3~n0QcmlqrE~F(GH{2k+Uhs`zW zlM}-hFGC5Z6ZxPmk4B_Scr1_-Ak_vT`?n<;de8}neo1GAblb{^tfroX?<4dSYv3Pb zW=L(ExPu3+Q$&srb2I8VTdOuuyb3vVaWC&$kpAIH2j) z&DV{asE5#E3!b8(lZ-bxgU~tMHTgDT6=$tYbOdWO4ex`tMyN7;L@Gyg*q2Ug3eJf_ zgN{JdT~}+#)j>40(+i!;7S8f)GZL@Vpy=N^eRFRZKhzK(^Mt3uKx<4A;TCRWa{N_M ztIHxlR3-+ubpbe2Ixf&foSwXthAerk?W&C+F z?`b!dJ0U?mq70^8Gd#0~`?56B0UT-}Y@l~T4JWfClqXy(hkqs@XOIWe%nmX|e~J}F zl@}b6Vv!AN4yOU1&9bTO52|#~EO<$th=dM=v{cibK0L*;@}+CXcy3*_@ut|j7G$I{ z2SNBVMlQHFa?w+|qn^XTt>uigH4GvqaF#wiw4J+hz)-Mu>oiV_VsUr-S|KIQpSE;< zhor)U_V+}0f|wot&B=x!h`376TrHAy95bqOHhrJ&RKu467o{nI4HwxFo(m;N`}&iY ztWI>CXPtHvQQTSC<0&No796t~cI4$}ohTf78*kRCP7G5et+aQ!P(Xzk%TZ((>+YB( zu3H~$0`YGP+!*Tevu}yzMKP^T%rCmPjW_~H-?iZ?B3{|<{RQRkg(|mOq?JjkZGJQH zagAA9ii|lqTa^+!Kp&9b_y9PBx;zeJQa0qyYa^c4&Z>On4&-6#-Y2IeJ> zx{*)dy1seo5X&omLPfksD#fk_bY=+s_&BUd31DC&Tk9x^?KWM~ov`Nr*r z5L=uB_&|&$?-rX_3q`@yaF!fgGDX>d4JWYVS<84xOv=_7dT!hAc9PnqHfK#|;er43 zm$He2jwSp5N>r=i8=&3!GgYxw>#bm@{Z^5!%>*l5<6J9Tx4fL$rC!%r@;Cv*~_b+9El z*NmW-I#J+Tvo#iF7~is1xv5Y;e~WSO%81%sM5Qk{FXAdwt}PGMoSY;E(TD4%H#UKa z^G;WdIyn+|D3v*2HPVHQaq(XtgyKOejqoI!^}hE}+ciRdM!_}pQk|Dh%)dq|LoHGA z3~MUfovMha-!waf%@S#l#LkBZ#>`j|uVe%FboNSTwCs(UcbEFbx{G>H%@s16Hb zl=$)z=B=UkzWvApwFs1yhFI&xD1meK`kj-YU+fqbSKV6Cjval?zQfAZS@MLV9GG8v zbAY}j^A7aEqfo>`Lzy-}PHi^m0v3IorP%gWgE~0;Jyv2|9jNC6kj6p%1x}w#PDQ^6 zKtH}-G@}9hNEED@WX-3!AV%gZ{&6jHjnDiuFL6CZpJOll&-VceB*Ysz!V*}?f{D5k z%Np6P6oh(E!yyL5tHdB}9f<)yTl>5u-@d+_`%a$5+lp@pE+72tw5!Z)1PE($@2WHs zKg{Wo2JE@wLCCzgNIsxiZL7_ZiU!tcV86G7HSplb6Ip>u2lB4@{r%_#EYk>m5mdL` zh~W6K2yjsIx`$IE;Zy5M!eM!dM_&~EkC(= z??K=9I|D9ZdM@f3M&A1X)qv`+uR7ov0RC&h`-=@#GL{_EkGm(Ie7&y#sOz)?f*sF; z>U2ml_2zb^A-L{B!@-n_s{HT;$DQV2P$8urif24=HzZJuBcA{o-J)vN+$0Io!p>9h z==`GsRt6>UNcBUL1G#%iDf9{U8r6RG$PX_8^JtpZ825K4*W|P*Eh2MLm z#;i5jE>?T+<6&|0X}KGqjvb*5J8n628Iy3Kl*A;`@kjoTUhi{%N09b;Q;xzlKF;Sx zeHZyzee7nKqoV7r!q^r1>uUsOYr&i_>ar&p&p9g}{oBuXD(M*Uv5i)S*@Z9eLsD8{ zxX@>>2=KZYk2Jr@!`47AYAlu8j$$R{{wjmD8wn2&JR(kgT!j>@MUMuS-Vh{{E+3WaUG2J!W3mR zI=7_|88dL=gA}+B+ik5ZaV2WA-9yC$wJT_EluG=)S{LAvlKPTqFYT|- zqIRy|oU)n#3ambgNllin4U3g$fxRJUlg~3>Tcx zMj+41u_ZKCJ6TQ|iBzd(FyWH>X8%SJ$da1s9qg)iN=d{t&Ae7>#mcy< z{oGFYG4-ehB59@~&o{&AJlY@_jxa#V0Iz)*4TZ-3=!_obc>veJ^xHhbm77}yS{g5G8G zL&qT&v&Gu#u&FJf3iOsX6Us*-z$@J4Y?PiB40=6R!NER~o5yKmGBkBCd?q>-V(x-} z-Cfmk2*5Q=o$FvUu*Y!Yk9lp064OJCi_A~=D#Ff+>u(}FJ$;Ar>ANY{j_Ot(wwx#n z)Bib^in%Y;6DOnJDGEa_{TWX zI$KZS$hNwF@7@Roy_cX1dNEO2mB7<}a*B?VumJLipuS!Xn1ytY z=ja6+f`CWDs++M5I%x*vtRyLZ-*XwJ#dXPVX8!X6mihP>qHxEI_8-jqgBBNz>X)!z zrp#%~b>R%;B_%w3uvcbHS|cN0{`=n+oqx0s({*%BwM)5&*$=-BUPNh;OW9262)PIN z0j~g@p*1A~BaASxL+43+KW3Hi79i{kOT!#RnOZ|P7c&wzMp9{wleE$?&kjgP51*ek z7XVep0)CnEq-78fQ{JlTx2S-drJ>JG!1HcC01NO0e5A@e8}rSBdl(}BXHYH~j0)R) z{EB%+SbYlZby9KQ`3@leDM)v;G4~ zME45-MLQkeTp#&F`_Lhl_g$#l1sbn>343L-sZ$p@GiRf%i!RH_= zk|bIYzU#X}rz4?U!WcHcmd<`728oH%8bkZG^;Bo`b3KUt-0OY|ecfTq(+vr-Ctk}; zuS_&)R;F2$fPjRM`g1Db%Ctg?^ai7?o}N)q?fXbo^MZR!wR#a@N6&}{GAJQg=6hY; zeVI^hwQ9g9(5Iasm+^P$JER*@P2aPmkJ07E)X6m5y>|?+qb&pt*Z{>$5acA?W)F=_ z>o}qp!DeM?kZy=(%w~7nu_~Qz+2>0=`H2m61b%o_q)0E)-xiaHN5YVTm@sM7*zLqL z8Wi(lNYyYX*yW}+htTE;2~K<$;n@MGg~K${e#zJERBZurp9miOcIHkDul)pbS!%W0maQO3 zjuCc=EW1ZJ==Ao*U&{wJVR3my==1pOJKY^!CH5C_)1av@wzs{Fh)hq;j8nHj5 z2~Cv*SNHCScfNB{&QCeq|7{iiIZRH*FPbzeayM6l{i=i47O06TWMP;NTSHr^qUYAK z&7n#sUB=crfKc}KV+bn_-Tx1kG#v2nI_hRDC6S|e`&Y>CN#O(xEuh|`I>BbR3|GMh zM^Cg`s&Ox$%)DH%QuF-YZg8}-?{xo74F#eKGu`cCM-P`qx;)~x{TR3lcZW*K#jaL* zUzpEI4%RB`fYe@o-x8Z@9Se^v+?dza2Yn&D!+n6XeR-oodA&EC*Q$o6XYG00ihmO_ zmOFnERcvZy41XoZS+faOi-H;nxl7+7$>sQBlLBfXC|noZDLWwL`!kV;M@czLO?h7r zY4X5X`<2Q@TlHV9`)^HN+vP9oaE0Z&S6?oFdz->d^q^Rk;s{_EvMS3ici@GTB;~8~ zUSrEu?C9criH0l{J6j>Gby}h~;9Zp{{6@>u#tATETnq4q2j@bLuH@{J=rsq8(;TmL z1v9jh+NNcUf5DSu#_dq*Z4)n!)JRruQJ}HGd_6WwPt{LpfOYcK`X1!@A`YljJ<*?e zOS0W0Zd4iRrJ1%2^&gB-(=|tT{Y96po34pfQn+*$_U)&Myos{0g6tr~(w9ot!rd`W zMpc76ihi}2ZBJ4W*l4FL|ISXxE=Zl|OOJtDs7Te8Nyi`rP9m{vqwq#6*VtT~C!NG# zQ0tLa@}Z^A6}=K?Fh%sKSEqim6ng#0NZalK80Ubsp6yQV>Mq0_2@xbNE*s=(sUFp~ z;mG0Np|s|EwtY4)&(vx;f5w=7D#g~E7++B!8KlkohO{om7H%aIbW|x)^TC`mNMd6I zn?L_@FscE=nr$-tvrG_0ZvSyG{4W@V6<=CL8DTqG@5!j?3A@&w3{E^nUrsll7vSuj z!(QLd)9vHnf?+1`_Vi}L9&q=T7BZrsCloan!ydEv*7lI+%g5vKc|S4BDM!-K9B@p3 z2O`Qsjtr93E=xUuV5l?c{;>gkXCu;`{KzB5AGz(?S%Ki>!Emgxh9e&&ZtH`claib= z{4Q>~Sii(@DDQ{*aO;B{^_(jZ+IZHXaOkX)*xNFJ)3>3WY`LDw_S<9zjQSf*PK@ZU z#3STG!7`m74MzF%pLd}#3_ApQpGQw#&NHDMB}zq*9UYdjjM*|x;k-U?E{3mai>hio z#oeR72z7kS%ewSR^n9^bu15a6;<0jWr|TZR-YL~`&}doF9eq%yE%1BUROvMs}WmQ z6-pjzwS{jKZ`R6Ap%+aoXtUdP`BhiL7U@qF=G*ocQA)0->o@e-vo+ULse8GTGAPwu zKv>1rnEtG;`X{ke^_^Zf@c8j~er2c~pk@&6VjVfoU+k%?vBLK9@=EctxebVNa^!k* zqxyQ@1&(}N0XKUCUqygtKE5_!W62x~k}&AcN1=MzGw45!CH-UJ>fR*4!M_?y1h9{t zcO5V1@oSJHSpwZ!zi)X!y0ywTShXA`T)uMqlaS_OQVNoe)7l2)sONLrvuMW@?7-E^ zWg&nfOe?D!riP&~(HF%+%uP9v*-{g3PpV=#5OHm`6ld2h^=ypKURM*JcQ9BT_XbciF#alW=?z zORf3$)&%G@Srz!H{Hw2oS;x|73S%`x+stQ1r=HMz9jeh(Tk&)U@#IO950|RT_oX1D zO|_v(bmMB|dNkTOSblYd%tNwj24P4+{Gk~+@lZHENjseg>fsltKopJcLd|1SHCdhL zd}P>`8uGmXeww(st!+P*p?S8v0M@}}%@x6(zaNY= z7Ld?8cDl3U{)sf!2`_rYo>>$bml_qgNMOVd2$p6(MX zPHC6P>wQhMwgj?cySdn{@-atb_C7#w^=jgD1A}pS~ZcuGW}hV_zhqx{1KXx zG#SL|{U>$RX6fLpPw*LJhJx|RBYCk6I%;Zs`20v;n%aPG3s}j^-(RcdktK+{hB zKDP~lJJ7wRu^0L0`*qZ?Wl_mrsF{g5q*L_fT#lF>P`dj#zHDg(1hs$8;_*+zwH>^? z!8}-LUfFJ;pgkk`J8d)d+bSPAECxAnP|_}?*kcZsIQCr(#U|WkM4E*b&g3$3(&F88 z!27m|0d->PCD8mLF@H^>+5hMhi`uvMenDCnysW^9HfduvuWW0s%Q zGmI3G*iP4bIYcum{N4aJQ6VSwo+Kj$mm=npjb5q4Hjs&=+~#6~gqC@I=Zuv1X9|#g zbPwO2^>Ksu7v~;O{(K7PNSj-W2VA^7wN1lmRPjFhL1Q^q`2y^Q4&wIA3q~A~DTaOc z3xpn2srn0}nNMv{bqwuZ#8?hr9%C8{sRe12Rq}H4BBRHsWlNA7JQ6~Z#dNs4R0leC zcaVMpXs-SIgpw@16U-oV`{GRB0qcT4zs&pk0FRD@neC8QV_v##Y3 z1(Pj;f1moDy`-yiOBaw+L>!`_J}BF4-yGc*NqYafCy0{cq2*-4c2=?D75rkGz* zhdf`W1frNVP<7D_CjXAn`x+k->{IK*nx_#OAe&(XydzW8n zO`=~oXpZd0hci`c({~TV?>q?7V{s%nE)DJppIdiz8tJ0V@4H!EA#;)MfXk=5DgyoY zrBikv+HJsl*k{n^#@EdizZWk6D1@(!UtDpHF;kTD?tq1^e;ln5nBX7A@!X0S&R^*b zADOTRUS!D-(89SgUa=LCku-H@CgLb5R#i_zlff*GPs`6tvEI|0Q^8QL58Syw z7Ds!RBiRr&7%ikhlx#&sq&Wwyr6ksA+gE-&?hJ$^vt5KNLI_gnfy|h12l3x#C#e{R zoP~<;wH%k$lEd-7onU%3=vdVZ3ursBtBv$xhOsC+<|m>sCp9CL&>+9cSO`KPZ0xfeVaDaSUel53OlufEc&YzjD)9^X>D5m^bxfFf#Iz%=OWMQju4L@g5! zw`@nd>9;YGJ`>n9|7S`&(> zUvr_UWL+X4{nBLg@IV)`cs`sUg_U;CpG^3*)UqAS`ikqt&}L9Uka`W=2_1?w{uGT^ zC^ekuxETa4bi9}cfu5;|%+%ltf6`!Kh-~PHhde7~G>FEVYieBXQd!@`g<{C;HqCI% zaT^yd&0d7a4VL<^OR=b4e(-j>y>U2Z6{6&57=wt?2}~?$Yag2D!1J@b z`mH=a=Y3WkWTy!#-sc@O-#y^JI%JB9=vj_exa;LdREniWG{(CQ(IQytHP#M$xjxX| z9}?uYySv^lvKF$ELFAaJvksCCY8uT0H*}+J*~VgEzA>3(NBHYo=xN34g#}mFLa0gx z@9p_Eh11%OvfCsS%dwjzEMjoY@e6ZY2+}%iTy&$FOrqF_f^uYng1Q(1(DaGt0b5NVn3qN@Y7T;0NE1gQM6w<6)cR`f~i`JWZxk>^)+sAU9m1iQHf? zB$YbWG9iyQ-UwOgom_c$KcFA?yUlP2Gj*eMrjP-?mDqXwW1}Xg;Bl-ugvmk(`|UXR zI=HSw147tSEFmZQlw-R(?jU}F>8YFdEY%220Uwi-p!DFRrhCODq`*43LD{iP5P#CXwH3YWE@?9fTD8Qj#LpM*~1e zK;JO{(w;QgxK7kOgdB(HriLg(LC#LXdVYwd9zqi_&ceK3MEd}}Ck9GJ6`Zm7O=Qki z)mrt6rdUdFPJydbu|u(P`TU7Hc6f?58;B*7GA#s=9K43I>cTsx3>Un77ClO$@=176 zsdIZ#+J+m*9h{#&Wo+1T3ts~ZQACIp2@g7`CBDGQ66eb!WDk+>8Xl|~fV2^j&g6-P z$+W6rBX$Kxh{yZkp{Fz8UBWHdr0AM;cfatR8qWN-K3sVd;r@4?n6|1x!PQseZKu)c zo|9)tr`5JP*zYN=^1C3(9o=!JK73KD%|LJ)u|pebmO%>2IpWgiW+guGz;A=df^^FO zM_bi2(tUD=qrk;^W&$viJ_+~ z+hv?pliaH!Qkn;2Uv6;=jlpSpy{i$miJ}g#6;4sPUUeLTV$+UTK;E&0x6BS(jAS6| zIDOCv-l~g42U3c#p#uX;Q6S>#Ze0R?{Ujg4uvkH+uWKe1Of124CkqH^^Wu3ofKti%{=v?Y3n)65HhC9a1i%89qI_X9ZteNr0 z3UtW$)NAUsk>8q%qhAdRXM_TyqHnrbqlzO*d^u#k_xqC ziqMMiVT44~oN_h>EB$J0;3h(wUW}eBoNZvxA6DjeM=&_%9vj-7!UR4IW!|bn0#(Vn z`3|nKiD#wBx~u}nR~YTMwQ)L~UgI5nk_;!oEEfpJ*U~QX#qXuwwBL>reDi?TdF9SD z`y&k5=}(9N7$}qzH0ZO$6`r9cs+Be70Izp>T_vD6b?q-_2B~#UDqcrdXJ@BxFSY7} z-K*RlXKNLE$E8)sOqa{lr0-Z%h=?V0gkAdH?t6+>p_Bgv{I~e|={e3d) zjNe@qx8p31k{Ow8q367Cg>j9LHse5*hnY32ZMH8VLh-eORou^+#eT%8hSnyS+xRsM z>ctjsjj{_z{pW8Oz93&1Pu)%t8s;vjgGPmoF`11&o2;#}j&yASA%n#z?`lPw&W9Jm zz=xlN=snneoAF$gx&xNh)r5Q!!W{CIbk-HLA_KNdf$~?i)!v5u5DCqGFcqv6_*|VhxLV$b&yXUgv~k;P z2O%50CC|^M_fDRvJlEd$$aHC3ZbW_`NJ(DLo9n|$@(}+@^#z*x2vY;LZ=#6JG=D_4 zN-gITvu8``*gT$A2*WjNV-&Xtw;tw)6Zdbq-wEk%F6+Y)^Frr6bh-RtBjJkn*&-jz zx)mbsmGqo9jhBg}n^hbQ_U%d!+qFUZs!!VA+mNYd#7-<+WMy~aMRsgmQj^a02d3Hs z$WeH#Dkh$-n?W|VV=2~03j|4g8>y(MrhS-h+#iOc3alH$}R)8_tdwmhEq_c(e= zT&3*QI5f@XVJwg2SoQ`1PO;eE`}7(ltP?QIULxpeQ-%E-;n`3-R0x`cJ<G3zuFUK+vHHXwyGoF;Z*u0H zp$*F~uzja{$n*0lR-iAFjqR9wMpTavH@c%WwpEFs@uf5_ZKy;YU0wj~b>Fxk8ARNU zmTHc&*bAgXt)9!r+rjJGxJ#rHO8tHL`o3hw`Y#l}-=5oZdv4D+{QMOF0RR8_8E}Wt(^#mYd)epHwcJT44^0WKuIb&aqm~2Nl;7> zIfcwHOIl$O`gjHRpF=2_<(KKIA!BSTUoXEaHHmZCX3j$iO^tYM$AwisQ~A1=j+san zs>6oK&R=kfg@7KL#2;GTeV(IUFSe%_&pV@i?)OhFHhk{Kr>Cb^?R?xG_D_2AdpG0k zI2ZB>lB880TYGyG_k$$QlYNTWiILRjFkvv>o@GlK-}Zey*o~EAEadG{Wkm(Hnt+12 z`KK&jmt5ZW(}PYxnCM7Pp(Woet?H8Ml{PcFx zri*%YI`^0?JTwJ(Ec_;g$6@|bz#yH6rr(C|t#(g#C`*vU*!bEva!heY-IBXHh{M;b zKID{`BlTA&wVC&Mizn?l(C!alJe-Ihj(Ih8glPd z>$J(-AKv+E4UW&4Y@;{kpRP9U`rlF<3VpS%y74tud{j*e3zmC+FKx~%O1?U*G=0d; zH`yXkWPiX9Bb_Ri#9sN%$cd#lgG@r@%&)n$bceDm7mxm59Sd zaPgQBl03W6}y5IBRh$GG6qBF@+@mHz7l5PvgB>qF{0&yqQ>Y+ zi7Ta*uP%o&@5;4`et5yY%>&t!rCDjV0y&H*YBPB}c06rz>#Wh6tP{EITQ|mfcIi{R zl!_dl3;7jGLYic9=xWA$8BjZQI!&Anm^u@9;In0%mSnRjJ0?3$!9y0G$|;q)cw7fP zXCo*8Hw-rkyYsVU)wUT0SBzXdpY_1Q(O{CVleQuDnAFrbnV80BHJ=sVaU4Te+EoZ~ zqYiTT%ygQhP+!Wop+vmS!>F-xWYD)wOW0mA# zm^N(_&8)(~K2>>4s$3Roit`?fa->Zry#ivl)7N)}c!y-`ypeSDMKi*eIuq&i!t{!} z_IBBEYNm)ptKcc=R?({7B; zrMsssDVL!!i~?DQ;jvqm0%Jy#2haArV_+?nx%vHb81HT}_<0i+HTfiUAdc8TDu3P^ z1|gi(c?ci=gsQDo>qM`vw;ZcSFDo0--DJGkcBBRg{c(GzZLff>tMBg$^P$aLTvN_> z?NvHF*0wR{!A0W%Mmb$?M-G$o?A zh=QFa zqofOmW1~}3RhoXRQ;j*bcD+q-oNBAMkJhI%I>{$5*p1954`v{H48jp2;apA&$=rul zIsGQ)e1}hpm&fth4!-KPIUJnGspzx+aoj>mZZ0axic!1b=BmZ8z#=i_wUUv#7w~l= zg}(%R2_qRb&~V81RW{02>{QAH%-piS`qy!ZsW5@c-a$T3HVk%}h{c<%=()$w;YwjulPL*5qi~$wRq;A^t@Xw>aO+jNM6J*4pW5rFVO9 zuXL)q_;9RvDtgmqLu6y>Vdz#<)Lk{J%LsIfnZ-a8(3jO`nR=)mDsYt-jAjP|wYPu)1g^~o^1gni9zALSwJ45geT z-QN=S=!a|OZfhw=aUf2b4NVlyAhS$ev5QXukNU{+(hMP*T24p@!YFkavAtvO(L`p^ zvAy+Uq>d@g3ds&BN!avERw+H$BsH@-Y}`ef=^JDvcmiXIWuL+;&}_K_X*du)W}&?+ zQgT4A}I&mf{X4QGAk``kFD_zjmG<1cc&V38=r9Dk}hh9Ic z+xlViDplzXLF-!3x2r6h`UIT_R(vnxS{QzH>{aD3G5zRY^_h&>HESm9$^BAz*MoLU zYpOZHvUHy^V=C<{s`0*?FuNN`}kqa{z*b+-ol7|u^;)JHJy`<4%tyP{mlk0J;>6fly zBh~WKDy8`nzLwIjgpnj&dB-G-xItbD;{soo?Z;O)a4fPWqlm}VOT^%|ww{E8`?Hzp ztUo3k?E22O6*$hb&#x#`<~DE3ejhK~H8Wa9cJF^`GMQX#ykB{1Ct?}KOnW5k_P#Qk zgb-?}?v2ul7F0|{y1yIVNdMJzE_lvVwQ7qAA~uDVi7bXva8kj!G-DCb;o(R_%6k zzxj4obU)bH>2v$zmqt}|)BMWGX6M`4{nrGrM6jHYe*1xxI-{Rw5I!AY3Ci=l9Lql= zX0$NAWL27-sP%GlKY7qZsj}piUHq^JHf>o7veea7Eb>R^_N(`hScRfybt2V!J-Zy= ze0R0Do3^Wpesf*&e7;|-ig|ZkdLDScz3*&)s>-j5K}FSW!V6nAqpYrycXGg&4qmoL z)X7E_O4?l>Ury89;5T-I0g4grPIgFM(?53#p#Lgs>d3gAXDP-17WF$ULnqn2#j%nI z-(X|lR!8m3>nwnIllYyc`IRp^qLlUhE9KRu^YEPAJ^gsSaU{sREzEu053+!0YjOCc+gEO|LSB6A*cDsH;sr4zjf9@x(O9fo>lQ}(o ze1mR`;hQXSH=CVEg|H!xz2;tGL-dQ~am7&HtD~KAWI0)DDsE%T>R{+%++K;I4jom7 z-+NY;ea|!meKHAwwEcFhW)E|b>1UJOO&+G+_TM}Qsm}~5;TJ>3CMpibe9Y=e!%fZL zAZf+D0Ro9Yb2kr z^swKTKAi-QXqxHjXGWls)Y&L6Bi;?ml}a5)qI0n~w-&RiQKIMjTlw@>YzH?v@EPf@ zvN~K4RodY>xC%O^eUiHlEFC#p9&2L*%&d9uMUjb6Yf;4-l@{N@n$Hi@Hsb< zuqm7S5V1NQRYH=%FWzy6&@joEzE%)NTGR#gMg^ih6V-6GH2U~M_k`$ueHE`ibpKh@ zANui=bXwHbXL%^|*OU2qcY7QCBd&GI&*`~^9}tG$O+Q7g5#9&Kjny_bH@1V#h z*=B5={P@I5=OR^S16UG4qd7%>;8){gVbJavPjZW>^x`Ow?;R?;qZ01zUR!B?pV|n+ z6T>D8Goc;j&^K(iR=&p4g0Y3-nB27d9Eft_XGO`u%#PY=MXYm#vqo9xDO(Cv{SmTX zj_5leSBNDRPKUNMpty31MlsQB5*>*SI{drrENcxcuc8JVwne%k-4qVleEo3c6fms> zS-4zC0lp#4c+GGKyQG&aSfpCWo(6}d|3kf)NUe~)1r8j{G(>|)gP8(YEQ6xP;erKo zmUlqaY{JU*0=0F6Tq_sln`sO=yjuC7Q@>;Xc+|4*Z-}J{nTQy@G-=0Ncq$-00-4LV zY=o+#0wa!1HoDY6mW+~K%$gA{@Vl=NL-H(pPrt1oDes4m{MO)lT})i~?yb)zTM()U z%W+goJ;}ZwG_0z3qD1K|>OAlNt{5!YMs}bHL~{Wr6V(-mQ~EAPE*cb#P6nh1Eb+^- zLC;GLmwHfJi*pMR&->;^6WHbnPIjRuKGX_&;YDyXx_cAF?mFm&QEr2W+~t*f(rYzq z4Aay%#9-OPi>P^W5-+Cao8LVy@QO*m**VZ^A*(jSpHVei5z)e6Xbt>oh22PJ)~Q1K zHqBY!I8h>nlQg+Bu5*^_sA3>AXc3{8$0g~x9p+|h_x?u8I*01w5KxiE zjJHGM*eBS@7DPuLqoBZLoe3XRzHkXA zMHKoZ6oK!8TAe#^dO_>xQA^99fWfn(B%t+qD$%O$7=1^n^veqjJeHspM%gC$sDTMI zk+CyECX^;dMI1SCgYvFY8f`7i8J0UaGPXMb1GZ*FQMiq;wwNoJ5a(daFCkXNK}eFS zc5*a7&GnTy6#*C9i}E4z%tTE}tz5CZM6H4dqlofuW>ZPS378nFneD*CQE2bon;QP;p>^Xz-*MV3F8yU3}nba&VAlo((Q|Xv^g)$U@^f zyESyEi;9WH5wnIu^Jv1e+bN7s^?q>PS_FXt3{zo%?Bw-ssKeQ-{>S=QI+0cGq6_|^i~?$1wUDlkwG{p5&P?c4_67lBoB?}EdO z1Hap?H6C_sY(IIod=B{JRbElcnTiw8ZHAEnL8droGq5xp@EiSwR z){fT!T~Q63T2;*t9vlGz3yQ{Ng~87s7+%;8Q0r&g=_mpEVUmOYy=V>)Vx~-2L+w(h zrXcb3DjK)*i$W+$cm!spO%c}MH6a4hCyKuUbCG{Ftnb>UOD@Kh4YOHmWJt$9dR@kxDGHkI zF=fp{^PrHHE2xTF(&=MBgyNS-JF)R*v#{3g#cIqx%9@OCy9tmehWi*8MKjq#+4t6s zwI;-KN>W6um3gt^^V5si&$2{@pRZwh$r%xlwMw>zd)+c|2v*nSWCrAer5e@rY&oUF zlpjPly0aO};=gAwn;bY@PT}_iH@rJ7p2+(8jZpN{Pgn8hx1IHnjgSO8KXBgIGIEW~ zN1SFlN3~rbB3$z^qy4->PNaiVlSk6T*E;e>L$o-46?%nKN_IIa6;cj0Yo_4HBH6~v z!Jp}GgUT!GCuOc9AKu@`20q*4nhH0odS6|qNDZqL!Hs{+m6bsZGBIY9PLc~X;^_)# zsSUnVimO2%F@iWJAA7AVV`>_{%r9kU511t(N`8x8qaC_1YhqePwl{Q+;q? zE`kW-85wukJ>-Pe<(K%UQmJ)(aS4FBxtyFe1A{ni7TMZ3TB}v0?eqnc{%_(ixEy)P z=~2YZ%cu3Jj_RlNP8A`_iA^Gog1f_0UpqLnL<{#OTNHc_H}Bv*bAlo9p**~9#(D05 zW~en3Y+{WLk$zLI^06DU#PTkC3Kc{6*?M1!%zJzHUXnsBzc-)+_gdD>J&K32ix zCg3{fO%0zMaj<`P?Hw)~^{nIgoa#I6qi5|@S?SOpTK_ib|0z$!#S?$GX~-^a6TsU- z_=1xObYm+Pr0;0l7qv zUo&Gj0Ur?_8LqUkM$A2HBLMC{6zS68@-gbv(YICd0L4o=G=6(wI5ofgpzuU(E{P$I z-f+J6ZF1XBp&{b6d=DA~3;G%m$N(8Q_&q=lBA^%sdJ+On^uyR^QndSEa7JYp2r!2+ zL27rBz#|EkmqF&|lvwJTZP_g(l(q`L>pbJvluVI)U$pI?dRY^JU#E@k!w^{92D$bXh6M6)Kz>qIgrngNf$qr;2IPUvt{Jo5pWJwq8v1TeT}@w#i*4 z)OgD-c1OK|4mv?K53=lZdv^3xZ8Pk`^<=X(ain6j?Z<7HV+kd}@F%6YnDg+@qKbk~ zRL2tD0fcj;nGg3}W|ryY^PKBVFJtRMI&pz?3%{C;ns@dGPCLJW+Z+`3OEAe%HBSWw zYxvWPsQCt+Fy3T0M@vZEE@s8hQw;4WAQ>tFEKD8CGJtk*{LGUei4HzwVxz;3qHk;&t#f~I)wEY+T z>O`432a34vnDH5o$BsrDzoR=w87rC*`de?u30XUm51~l6Rg1m)ct{Rd+&kHuseq&y zwQ=O4n~rG6q69L7u{p?GWZtLNtSs9F9!vtwifwd9hZpdlGRnqx_gLKg%ZbS!S%)CP z;Oo)IJ_N;XO;1Wo6)PotdBxE*-h%dTej*EOeqzxOG$`jqp@ckBuY6+!Cxk0H(HDwQ zSR}z6`z{LY9LmzbCvnbO))%%maZQ))ZYpn*q{K<?^mc;-og3$PT7h#cwecz9Z}a@adWhTRC!s`II?1*g#?p6JhYn`SBGz`fI1*? zSuC_%sZlY-GkzO{g_;18nl-NGlPfyzOZYJ<4hG8Y&9>uI3lTnNc$VWZlxyiHyF3Z! zag|~BaA~>INbNbZC0^0ixJi=?^@sp!cX@y6W$rz;wA>$e2;)=!>FmYVvBqSBpz5~OdGsXQd78xNDaR7c-r{D-cy5u_qHS9l2 z03rUh1P}qVgs$2&6*tAQqQCfzAp->l$An8OgC>RjJPhASiL;P~xAS}zX3Me2M+Dtl zH`vIWvNzmAAV~sNyt5<{yiNd(Yip`3c^4}7=qzH9D5tIV>wu~^inh+r#2`$7ooH#; zjT(9y%Q7eUls9l>I&=gl5Pk52Aamg%1ELz{xVV_Oi#0)fGFHT9g>;SNmUYFl2%@76 z5?AEDc)?Vz5O0smf@31iU0<@coyPeeZQi=j1V8@vu$I$d-x|R~^z6@k0h6EV+V>N^ z2^nd2z+m|s(kgbVEoPl9dYS8BNqSF_LYS1>hX`>Z&d+y|5t0afRyAKRqgf1>YKc|+ z;}p1%mN&-rqM#XGEs!;9HFR5}8tgWr;C~rYKPN#Hj6DjSn=o@(8<(abT7az{u(esJ`D*C}~tH~l``v6x)VTaP%rMBJh4!mq=suFEHe z?cLtO$l$U6c=labL7IeJ-1AZ`rCUE?etXlg{-7TDX z2@i~$E0}dR{FxOFqq0AN`9#k^aia;r873ez#d2PaLg&YOv1Wt;Dur% zdv&1Ij`fDd_g@gBM4>xTgV6FANdrqp;mGK!r2L&I*H1vofggG+pCXxFBY!dt{0RQ1W-{=y1le)fsHU~3EfT1-2eyVsc;@XldaLnVrgN~H{BP{3xx98b51NM{`^j^p+S}BH(_I( zc2}u8eL`nDga!Mk^jX;{JR52Xks{Fvvq*k^Jd*Bg2-8-*2EYmXBCax@tdMS?mFzGGm5$knQ|D`Dnq~IUf7sC^qJU>p03gI_R>w^+z z4KY&N#+#`Cb}`FwFza>eWza{NCM7-#Z-}bXFp&J}gE5Ejd@GGTyu37&*Z9%09s6E) z;=Jsw8LL;UyF_(g#tOrIj&0r!CfPz_V}1Ejp1tS(V<( z?6`a8L(d27IwZK5*5OGL@;1@wubZm#4=LwSOV=r#^vjjuRfnY^h7 z4?}I~S&pn?6m#uqy7x8|I0DX>cDpP&5Jw6f6VxSH_Gu{slnKmc>4a1F`R%`>GV1X> zhspWW=w^;+Rh=)HY!0_pg)v5yb1a)j3PG{+kGZnavz7>&^lokZ1AdJuVmow6mYf3C z7|%PC@@#&;eEq;3Z&Emea<$sS*U7>5E zZbPnM$FcyHT0|UK!gKA@>s?0QtXD6LyVD-{dCJw8R&Y`g4wdFFLy8&{{J%xuog(<` z(n}_NZ|b-OGs(m_6%vG1aWQ_-{w&#i0<1xNuRGw=H7no~TeVa@tH@xex|xK9DGW$k zshgEZ<({~BC`U*xm{c?#b?lj+!|a4ps@8m%#2&@w;xc>hM>ODNVi54KObP^GN~>yO z0SJ!P7f}ATvRA#rnS<-_N4d!UJ15)PV-Sm$QU=LP25d*4XrHb``5s`)DWhURNvHGQ z`q%>x5zhPTD%r(w8^j18%_lU}`UCyS)*xZ1HP={l;|$K7E9ay3uXPp=lrrmP3^)3a z4$jCil49cjHf?ZIm~<&;$|@;;aW17$q#|@#zfUWU0G1RAo6qg*+jbx zemBWk4~Gm!7Y6Aa3V#8L^9Xd|f58eKIu^*$jDgddjBkbh&uYPIQ)@lJFAs;fLQoj6dXYX-d>j5iPp#SH~=yF3hQ= zY^Ipav=mL{8MqSwTED2ESFL(oP?p} z$eLoK2*%Pk`_cDeH`ntmbZa<>N^qnZHBv7gPPA2vbiM=9l*<@|{Gr z>MVbZB{oE&2>69%{cP>?C&nGZM`6>sIrPX2jWU%5ImjtfNXXQx0{K_?Ns}OB_p+FFRG+$Xp_pS1avb3xFV1$18M^V`Res~Q|mGEDW$k4&2 z-_g@TmKxfO6DH2zBKU8-?O`93QNJxH9r;y09Rl?~I?sp5W3?P#px_B>yfRIxZMqBl zoZ|33C7>qn>M)&MAqUNnZ8?cb=KMo#M#NU#JB!+B%IFzMuvx;2XxZ4(!8HTUjKR~% zTjPG$nSK)>v0(f4ahxmp^4hB#L>%q*U3sq%Bxv=3kHN zD}#k582sfjsj8ZuXOgxl-7l9=*B#jneLqi#7t%1()ww3{CoeuYZ7&P5N58My6_G_$@oCY--{{faSF8UfHyeDk}n!hh|EQp%WW3-3Ga{g6&_zs7zx z;C-Vp46|MP>g<4n;vnf`Yoh2I?DoVbtz?z8`8G%0GdP^>H?%r$Ny@i)x*#~uve{9~ zDq6a7uoJ>E+CY9Wp)mY#=#u@?#eY>vsZKWBARNSJC|EP>LNV3|yYO|@@{@4OB4G}p z&!I3d9T{ftyaX8bnTdH`7UZMpVb7{T*U8O~;;1pMt1+Z2*YFiunI5Rop@$$>0>Ti> zkyW|-fU;XajFj*KT@Hx-4P7;|@Vz5w6>`ERm&O{1?AI|w%j;4r!#CsPzFeatYsxw1 zMx1dNXZ2#Q$Kjli3jpP52_%juNk@j`;SU+ZAuLNdLrEDCIAwY=Nq7;_SbtJz;Sq8` z>{YqYNQzXY@xxmolQON0ur$wBrD9&9)F|{?J@(#9HWfwcW%HSLaY_5j7wI`sOHM7B z&5=e6KBvgZpw${l=0I}RZwAQi;5Lz*6)H-=+~!m*)y^p4O!uUVW~No7i(0IxMhlB= zqN}#)0}p2k#Q0YVYO#R~_07t(GnoNkYjuf%ic5fY10Ezl~lj*0784hc~zTW38n8Mpp^nV1;CS z_qg32V_qfCZ*#G0&_c+GoF`I~qy!eZZyg`ENe`$bk*LoS624zNovSQ)D&!|J zQsn@gB;S{%_HziNf~GrNm8B%p1u=Q?8qv z{|Dw{9sa?5b2Wf1#OW$jY^Kr3?XP(l$@7F+@MiSjlcD2p=Zho5Q<(2IrUYr{*U<3JZc;TB{WLEF7R%IXYXf8NlhsnSXw)`Yy zvU-=1_jzgL*^%cJE~oT+Eb+Hj^$fM)6=HQMc1du^L}Z?-_;ZwU$~S=s;c5qi0zx!* zB)DZxuw~YVZ$y_%l!xD}x_;AliSKqtLgG6_SJ^vMVEzn+#9^^~BK_~P>-N!jf^b$D z@9Ye=+0_tOa*iPB(yeDXL`@CR;_>VLAgSQt>ZG%0@^8%#U8_znc?pG5hS9(Tu-{Yj4- zpjY78+4BDeaYu)5eiZfp^n1}KC%^cVg$(YFKk336ogQ^Rz`bG7tlk2#wThcNVs{Kd z^&$F&b0Q4%rK@VT515<0DzOF>PU|f^tzIV1(UR7jXYsNf$${gr%+r`|KfYA|a;)ky z43>FNo;+);p)|J$hQUnB?G79<|AXeu@~wql;%qk{=*52mK`$s={wMT;5dUa?B0V?t z)X2x@A{%=S3r8y!z-szt6Af=aRLZQ_dt~6T8W!OPL}1PZLdgEn*`z|Hv$o(^s%rFS z*g(20^m{Qi=#0NY=s%y68g{JSSqsqrAe^euWSy3m$-juEIbJnJ!!5197 zT=!7jQC`lY)AP8do#)>nbZ;|pcL!~| z^93Ibq$X}>7&jGZh#>?d0-?f&}?$%6Va7*8g0v8qSo5RFiXx*wTEicI?(!;e^R-I|GC$Z*2+%M2Q6Qg!3ukg7%)w>-l-sPD1N~zPtFXoS=M!;Y6AO5yOZJFY9Oi6ahGi0l+HKo`5OQ z6dQ7dBYRI^zbhF!{fIL*!Y9`n zTc7jI!YdmkoZ~2`YYrB1fV|6tD#vxm`=bJ9&_mN}@2vqHal+$BnQ|5V&csGyPdZ zX}f>0wuT4(z94dK`oz1q?LD}1?p{_lC^+^NS1-+JPrN0WjF#Hh3Nn~577j!2i)YM_ z;W1xQ`%6sFamcX`ckNGl~_7`WOq}V6?LYg5ED&O(<(_+`AV;m5W19+JUptAp3?~ft_w)#m2`L9BLaVo(Bh6FJkc(Qb3foj?JR` z*|ma!sgAdI)Pf(#Ar~wzq^#e3{(21N*S%8_1xE0bwt;ihs-Q+m2*weLKrC`GCDVo5 zf9bIGC%;@w`NeY#jueiv1?1c~bQ!%4SA(^tmW}3_Wloj2aN)jGs#2<)3II8ZH`&%DcMZ-GxMHRUs8(^>J}IEbfxf)ZT|d8=t#cEn#++w zD@L)?8B3)&KQEgfYW_=PPGy=Bq>2YbdhKUBlC6QzHLTwsBox*{)aT7Zcq%SC_O)`; z3F~V-8~2i9$X~lxdi-Ymn}WaVv^LIeS9rRzVeOz`h+W}ZaVxQKQ>n`48Pz5!vNZcE zvOGp&CBTofI-%AVwn8&nu@<;b7%?1oBx>@NWz?^cDWqGKbC#1W7*WZKQtxUN8Eu)n z@+U15o}lxvRE4{D(6JO#X;z{;+1S+p=#{r`188@S>$n030|I%Loi-h+y(2n$kdx)^ z;E^=~%pJ7QvWm~;KDw=~Ev&47A5f5T7&g<`g=7sD>0sk$RwwSBb^~Yz=U!%tWPh(_ zSS1svT8`R@HXWp|*5gc|fH8s#lVE*sfK5-t>z-NP=Rqe!O2fB)9YL{d-Psa?8((nJ zNT&b`+u!`zY=aOVyU<~aqWPnGkK*EeRGk1mPRMbf)dSLG$YNxdA_4OoS>PXDWqB^v z^C2Eoo<804yC~fWvs~WD)Bz{@ltrb%VPn)|2?g`iKbPSOvW2a25TeTVb%~% zYQe?$rpLRlhZH4hYKijSrxx&?;k;MpQ<7L0C_%AqOr*Q(qaX{VT55M46mML(f>_p6 zLdEZMJpPcsb99ja`a%6Juuio_R1)MqT~|)urh9akCv+hdNHX5br;5+r@yLAlG0=aN z0*k-Qtg|HlFH3>BIREQX;J+bO5_W$RD{_gKYhIO-s~TW)ccK8nou6fwr#Zj^E&?ar zk$fg;=&pYJhsXApZGx<+AuHsV*ofYC}Y5#0uC z_{16vkx45{tJ`Noc#Vy#ZY}dDO-Ji;f+sSQiGEA7NS2XMq8|_@DSys)D$Ie@&ClJ z@adH=%YUB#lVsuYdB81z7xxGu7Y7`D{m2OM$1DNyEHWWsKYy@7`BrE}MYWoGjS}ED z@ikmE?0 zsB3*ULiHwD1MirF%yLoX$te?PIbMXHYq+Hz>0>{p6V{&T7ZHpV;>7WhtQRqI~=q z7D1P02MvFw^y{DA{8&JhYNd~huU0gaA%KkG+lgKbrb(oqh59~-Zh4FMJ(uevtL*{x zfgkdv@1vibMc=6$J*tp$Y<{&DLwfQCaa|pC1XwpD+vv?-4vr>d5AjddqLFwoNX?Zl z#BV0HL@~Iy%fjt{l7oBx9sgjm(6Av4vl`CM?8dy0GO*TNqAPNym7v%PlSwmInd^$& zU|q-Gz1E$=yM(!qq6bLf#UrBws0NmN{ZAcegL8v1;2lPlveuDc&V(8n`09`5TG z7YnL)P>}J19Mibit$zk_c<}Il9MizS=78y9Xm72w|D%)p_xZ&Cb;vBTG2*qEPk$X} z8YybzfFSpem003cIcrOs4f@X1Ygh#PfUf27sefI|>>?r_4cBHCW5h^DR6sh=6`%G; zVhH;w7q3ImlmNf1d{R~81DzKT;a^e^NJrYf;-Uuq*Bf~V6jXuvQ#gGe1|O~ir#J+@ zL$@3opKlGJdgpb8I{O40hurw@rx2>{q>kI2n_2w*^u) zSNymX=Af*U9AOnOi!bDKI@k-5>2bOec^_}mPyfoCS21?&EJU40S<38|b=Bm*r;*Hk zwyJ-2gd(Us*$AQ6iUF;#q3gw7JCw98d%-ABY08tW0xk<_JnB> zw?bDBIpXLgG=esyq{%#ClAc0hGzbFt*lF;dQIZY)7kjndAUK2{nEn8$AjFTb!eoDT z*yf0_AkvizU&#UY&PUR*kq2X6b#!MF7Lvjl@@M_4(f2g5U0@1it=DsSXwVl+ML6ga zBm1I+6~)AFg@%;o?XJ#ueo6NOnZ9V?HCc_3RB6uVZ4Fnc?(eH+D!7cEq?%SW>K;() z`?NXVropd&Y@2)v9C=CEC^Jqf`=xw~r`F=;{BlO}{wtl!m+$Epm#^3L`tJ~_NxC>L zf!qdeou)KKhtB_-E${8p4cM|E{F=dz2F zM&XLLBIK=N0h(z1TyX`ssJ;mlPBmXcnD-e5Af`D0sHYk}s&@q<{$TE|)h&TSOwFb*^m{--UfrKrMM z@N`2t<7OFt9HQDpZS902I2d^Fv^wuYQ8@{+n3F56?qo7Go(6dmWAFfbb031IW8uO0 zZ3F}N!vmo7Y;H!yBardoPB8jb+Dvs~`~EnKJ1i=8en|Losa_^TQmFLm88d$~UVsj$ zp<|N<8Vo&U{UldmzaRNEs@Ft*W}0N-!-RzFP~QAdJ!xY`a|pk=^+;H%g~}k=<`2aq=6EUpaPU z%I5=_YA&g8I{JEke2IY5H>xQ3&V2ChU#OE7MHq(UW0PDteBWRSDcj_xsVt&oR(g@( zWajaFDP&SYa@+>5&0)ygr)1Np`y-v_;miD*fi%g_F*A9X$W$1vo~6T_KZP6ED{&6K z+_-sy4`jH-j^F>z^cr|jV0tn*E~#0}$B!NHNtB_dquG9Zoo(ZCrA+BaNf#jNXJVc{i1i(7m!TD*#u<7s_n zx|_Q1Cf<_a-_n#Qglsv^FZ0QqQX}C}Is|s|8bImZc7htE2hfi-+Lwn40$<{t+UrNT zu%tu$Ceg{h=PjL?v4(*O-f%xq%shHuYCdxiV&l)36+?Y)^q1}z2xTuKIVeKH51V6> zAM!YqsIMhY{*m9vo<~yBCe!czHhOFxH;bw*@O&~KQs*+3WaRGAoj5e`^`L=8H@mlx zQb%tHK9?(RPV@;FDtEV0Yih_fN)Ue?$9__i@v zRORQ*XF|2dePw_7GyhRld)e_LUc=3%_JZ-}qUaB?g)~=Yacl%t{|Q%CdkjTi$rp-6 zn%^`3D0ckmRlY)rvQ7Wht5lw__0$`89KYQSEF;fPaJ}s-N6ic91rkM-lygQxzteMg z-cTFoHxOOP!@pGls{QDg(TX0c;AUTp)8459gPGX_8!!lZaL@>PrqEz=n3m9jE#Gi& z@Md3{e)O}y*9BU>q34l{>I(lAmw-qv*pojQUD2b&MpRoXXq*qEP1*?ExvtKavkcOF zWv^o_-xv)4>~!XEgEm$+yeOQ^Z^Z_} z1>i8AVZ$(ZfhcDHPR~D0Z)OX!0*%%W^=Pv%`3i`jMjrfsZRB0mdsnwgvoede%{vZ)EMFYS2JBb<97rS8$5Q_1a+2 zYS9PcEF*Ae3%*@Pe zvBk_}F-sO&7Be$5Gcz+^F*7r(ufO`Ms%NIAyL!#jyhX;18xgrO^JMPWd!HB>nv41b z8W;pVo^DSYjA`)xq+1#Ax1tcU%be6R&IlVLB$)@`yc;(-GKYyRG9gh(1BuG!vy;oS zb7W!P2y;a(bUG-q)R}uHg@0FfwdKw9Onvw}{n3@nvhG~^*MothlcXh&d{}IZ4Ll%+fg7Nh^m&TtYR{tQQF7A^?%UFtjzM*?b(gF6$d{&$UCD6tK5J zV{iQ8h_nwB9ie!i0tyo7ix*`NhW5Iwi#{B@eolY(T`t+p zCy2#x{jfK(>)<(XseB&0ufKd3=U=s=;McFz9#05salk~fjQgH8FI_CBDOfBgY=iQv zQ{M*VU{q(?Xgpz67sGLX5_lvQ5{l#u4s^N-LMbBWf*ipH7bnwT{=MySh%STW(Qa(_ z)`#27JpEf7nAGVHMjHbE7B2h`7}YOIYtA{ByR}=wpdI%O|1~Pp4WjsmhS1Xn!RG6( z=O_31qNi{8TV1_6*7G5d_X&Rk{g2zLdsqNS(|GI=0H^!cBr!|+e}6%O|Lu1z;opXY zGpJlk)HV2L98+@@{w1ituugp6F~ddXIo$LJ=pSUc1NsMn!p;BC!e(RR%$8+vh!|CB zOEM__^%N}p_oX0=f6rG;{XK>GA5KUCK{6BmU(yZ&ASvL(89MpxsVQ7Ta8Ilg%IPv5`Gbkyk9SUSTNipU^*#Gw=)|0 zaw4ZApsQ477IgR)vdl`dsG%0ZhtP9@xr80#`-W?clrvGm-57@60Fy;Y=o}|3I}1{MW3EifVSLA=`g{DeIr}ol*5Ig!}u-|A+&PHT(CasUg;! z7=p;7fCvge?@8Jp;bTyO1d`aKJY4egI$^D#uJQ_vVtwX4PHXHqg*gC4*2I8)vt8xK zccO+zo!A5_trX&!6IXE+0@TdYrm{_@NH<##Us;5&w?mE-wt&*U*I}1aDmVFN*BW>-a9mLdTzg*;?p&(*dV^gVq^aj=EC#DdQzIKrmd+hAJ^D1 z-Prx{$jLbbZ*u*XfOLL=Pvm?tE2mCGg=5EPn+1Zq& z;)wF7;-bQjrZc-dQqxxSae$}{^Dhsf!2nWF5>$@Cz?fup`^~!UKkg;;AQ7%*NZi2+D;1fzjF2YE<&{1R%?DQxCmOa7O$F?R53sO4n=eV zA{^2hj+6_MCK>{RPa>=$0i_7^K@f}5p~o=+N_npTPq+uY!a*SSpiU0t9#%GiaO|k% z5zpZ-9IL@t0wCB@u|D3wp7g%Q`GDTuAZOuQ5ZvqHRE<=>npL zzDCLUM;1ZgO@Ubdb%RDDADV(=E^+&Xaq1(-5kP=Y=63fR{puG?zV+=cv82>fS^l2FB?bQUbau`!qE2g9v$ z{wMs#w9Dvm2E=c7%YX5k2kn32x4a=?Aby)MI>kq%1aRoT zWl7vT*=BOD^I7u4^7LTj_ZI*kb?LASTC}KS$ahPw7w02#{GYsh|Al+PP;=~K`#n-d)u^gC3g=IzRtQ=pOl-{=Ki*24Wdfji+9FILTF)`frB2iZ{4ELu&>1QDw- z4pmMtj==F6G1Th^W^lCUC4tj8Dr!2=6w= zY6TRqQkcZ7a5a^;4BYoqqnv46C7s)(C`v5{w8%stxxW9mgp5^5PDj5Xxv805WOaR5 zG|Sn|HK2LsvZ(Zc=}V?)(^_i7!%U~VQ#Zer%HLW`^pOVsZrEVTp~r05AXjBgddP%n zit6e(clEw8dUw%La*j~EY-{V*D;BMd!D10J=O;SzqZwmZ7Lc*L=U`QsxT*eaV(Pr9 zPQ2#qpd%D*yohB?&GfwzyqWTKNa=q4ZF-=-d!6DGp6D0{ zTeq^OIy#Pcu;XQeB~5>msXSa!9J{}7zL4x`eChjUv5>d%?RoA*@3XDo;Zs+YNAkmb z?wEaXqV~xWvmzB|i^SwR=DEA}(gqB%^*g0X(dWW1)6fp`htz6UCp!;`ICY3sIx&Vw z`BEDvYsIbDj#h=rGG#}dfb9nh+NlInR-FYTS+_>bo#6rJRFh#8(8Wr*{YDjjGNchV zwmqB)DDkKr6@nnsA$bSs{^1=dO*CvX>fyvmr|+od#}Qa8R?OKz&y9AWZ>K>@vuo>} zXJs<7u3d^+;GvXSp$pGAR250;?N==bBWQPDdtnu3V-n5deD4kY+#l>m;I|<{45kPP z>m@DC6jf+ijW{$7fO}sLzK14>&+GM>Q~u}i;pj+!r5QO*p#{04W&U@wt7(5yZSs9{ zYtvPsN8}~`+lz(h7?rid&?5GsTHVSp7BOFHyJXc9vU%CxP2t>BJtvR0%c6TFT2iUz zzeSr}Y8&6Zq|QHIZw{lxC+%B*W1tlqk-9U~t!Dm+nL9gby|XQF|BW&GUJ=F`9C35a zjE{BogDu>plAnS1x1#1mBr*jZM`LTov9m?tWM(wFj4wX5+R{aIB}z%sWN@4@2USlC zhQCvcB88ZM+04$V0mVo!J?NF1w zo15FyI>%=7r_bq!r`uh_`(@f?=j+4O(ZuD3*ZcbFWhWOmx5w+mB|;QJm(T5i>)ua% zQ0qJ;3u<3ix8&J&q>=U6%lN{8o=^`2eAQ2H@;g;{aYq~MZ{d@uxy(B#{1yT?+Z3a^X;=ci^W|LRvOuOE zQ&Wi*c}pQqL-0^;47CpN>U85Rp3gfFYqZsD?;?NR(s8z%?MxdlyB@k)O`Z#`o47Jt zvM1W{))_5QWmA~p2J}P2DOBhzTn@~ZF2ru5Dh*f;rr#6jWW?u2N?q6%3Qp8d#e-c z)!q>-Y1d%iMVURxu#N>SxT#YUHrUHp{3mjQUbJtQq199aTF;+a=IZUmU@Q}mCi}ab zzOJiB_@pm=sz}-fc*`va+)~4;Tc7d&1hM)hn5+NXb~~rHq09ckYlT;#6~E5Fp;ov0 zMm8c{a0;_9AU&}nx6y4~X3%k5AaHt*74{)o$11tglTt-Rz4CLR<+n>6*2C<}b8+gq z!+3t9j7!?Bud1^;cL-4aUe8C%T%?;+*CoinoqG_zVq9XPTQ-G}+&2r$djm`{`$mVH zrAe%I?muuWOAj$yDncJe}vf zWU9Im-mKQsGEBQ}-Ec_imH1Yp=n5Dvsn|_A5_TWKHf*23?VnYgh~vRSI>WBUnw> z8zcVIr}cFf`$C6ORneG*?O1iH3U`30BPOlh{9^P9lgo_jw#A6k>q29Ise_7Kz&g`~ zUl1-70X9*jRTuWi_GqGRN``dvhVldNz36qT!bVJcd8MCf#9^-Gs=6_Eet&%>dV~mp zH~NNXV!g*Q2cUeF{z5gWr78vQ9JMrFvbkX`$i%KBe*BD?qrxWgfr(+o^pS3lsAv3p z0HBm>@1^5waa=N}jitG>Xo<2Y?XxZ7s3~SGJ2iu>ii|bA(r2im`?JMn0r84!l7c+W zN~KG!s`No}*n-9Y{MVZ`L)h`2HgzqUbScS3BcoOho>_L{#Fp51&~f0(qF zQ=6eeO9NYx?J#WGPvT#f$>1(`Vcs9ugL$;DToaEY5;+sTK5kp1`T?jt^Ab7dpXYI( z^R`x*V!Y93jAOQ}tI4~rEpd?Jc#8vGCJ{sMmjadHPqg`2IU(ZZYD0@BxUiXn%IY;@ z6)j|pbc>hzW zr*6Qg;fh6(j0DZ_FglMEQBrqN*moI_(MY#+} zXQFrIo|G^}aiL04-YzRW8s0y4TUqKY9r^rhW|AFD)(*ujpK(bIlG_jLX=P;tP^t9WJ<%U1=J zOuVp&cgZg%*|v`Heh>Y0^FicG*-6fo6^AgJVrqmMpb%~^`ayvG;di~?93XkWHD80B z$a)-AXCm?5e@nAr92Ty1D)Sa&V{us;)5rHADb#*te%VD1Yt2rep6B>5Of?uA#H6Q& za#~9&@UAiS*`BsKjuY0{sO^uCm9ochy4cD3P+VgTihzK6E>cQ!#(dH!-@$es@$z9Y zY)f{)%=Pa0N^S}XlJrMKVC*N1@CX5!Z~R*ls`Llt^DD!r9H}p>BP|6O4_%->q%@RE zwjAN|A8V{01b=)9C>NKW@vry_LoxLX4tBcFfZScXl$fU$^O8m+iy}ydAW8LhSNqWu z(L3gtK*$Q%R#SQ`<(Q^dN^XbLC+{1E7?#KO?lnC%jtdkFG-B@pU50Dap-1K?-9ePj zB#jNxRnHV6Ra*{j$(pxk+4B=dj#Y%NOs2G1JN#E!hgN8>FZ6cnrpNA%vNSEhs8Kdc zI@ve=Bblo|{o$GsCE4r7-f7E=vEIfD01K2>s_W&8%_C+69reAxEZI2Tj|N8T{Nlu| zB!10~@i4G|))_nO5QnSP>fg_0?tA|0!)aj@@#&0I4Beh>#Joa` z?Xm7Ig+2V7*-WHN97slo$5xA*Jis8zxuew5$Pvj{6xLDhT`z$3%;Lj0KUv3Z<)d!TC>3Bnv)ILC>;sKW?o&KA^C*&oXI&EO(+PBQCcPqKUrFxX zc)F){IVD?OfRueno{+qq{djO-VvPt$šnj%Ol{@EHYP$SjOhSU7W&#DQ1v_#*E zEJO3FsPNgPL$lX{(hVYx(KdO4z+i%~7XoUhWJ+NM5!z=RB$2<*ki1_mu35dQj0M-=7O^9ga@S|!QnJ&jUNm9xX9mxWt(bSq;RaDt*&$m z`#-F8q|lWo)8TPBI~Ccs>E*tF8k)oQ@Dfgh5>;3rz6TXTjg^e0FY@LVxyu;|L4=}YD%Zt=TR8ypK|f^D-t(P(BwW& zfLZk(XW!JxhMNb4k+SSwuX7efHs5i!OS!IMUdf6`w*EMOI$@-U!V|vF*z6nMtc!1s zNz8yF3U&VTIBzntFg*}aW>{CP-MA)byB0cCm3y4_mhRVOaXjrn;K>k)4}tQb$Z zy63Pm$ayd4wX?HpOSI~^hX6i6dwVpp0QA8Y{<&uH{b00Ev%CJX2E2%6KX%@U?wRw)kFTPSz>BwZoSH$y=f_a6i*#l-F%^(5)Zzs=@@x&MucD; z^c*ULdkPuc4M+gdPh}-n7Nxz;V}}9WnB${A>iiu~C4Qb9LDJ2ruep4y!j6kw0Ny=f*YlJ)f%AZIuoU|n607vXcT|WC#rEu5`WGwsByZBI^PK?eD2AU^U#s; z{WWsMb-uF?s^PLO@NX&a;T?9mvyZ^M|Dw!BV0?KoFWi}b*e!MlYf7aC>H3P;rHubQ zAgsv8`Z^Giu2rDc?F6$GnD(Q1a zCQ)O04JDETmK2iL$m|;N2;p>bSLkiX1mwg1%`SMz5P5EYz-d1!4GO z&~+!{24*1R#*!w^keeRZt~VX9wgFgyIr!?)f0zs~mY(9b6@WO`lWs$)CtyjdpY4*` z;~clH4d5U(W!DgEbC5DLjbk`1?H?yk27s}ou|m<|10O@+qgWl77>ScpV8oSEKvGDk z$*qQ>$<3Qn07{3i7g+$xk_McX!Y8>j=09^|tQ(;D>VqOg`f*iT>Pg_OCHO_M*5$o zw0R6HcLE2T8j@42>;wm#06#4R$Mo@8dTpoVU1M;*xl`aFu)lG8|CLA8e_b2=hD$~G zya#+uh=$8zZ;A6)2jTPAc*eg|(2#8_{L%e)9{Ezh4!pVK)!rmo$nQfA?h;2Hi3zT> z%65!zW`)kdhicxxq^`c`Xbt!%&@iJt!gfBa-w%zukEKOnz$6r1@WM-%{c^_Uu!c-< zmhQU7hCB$XRAXOu*Wz5)_4ASSnG^`lYughUwsGdJ_OHX(YgOC6{|JXlY{Sm^cxsKK zPSf2~&OD8JnJsB>flf79MAr*dVxiEx^%}OLO?Sq&PRyu3joSG%x?Im%ZLa9N-puL( z*)`Rbf|mju@2`5icr%yMQKT-{iYv`F#U1?V(TVQQ)6s|3C##7Q_`t{0#*n!Hmr<-u zrRR>*MES!rdG9!Bu&0a;?Kg)eDMAL7$&l4!GXaq#VztaNV#|yX*pfG1#Z$E`=0yyrkpW3Q3#v+Wlc1< zVn@9M``OODq8I<5F1Ki<7i-$lM@6>c>zv&&UM)|Q?Sz-`>;BviS`wRhK>Vs1~7!0_xnuwH5O0iu#GstRVEO!mesP3V>HnniiNta z)@tTS7Bv^(dH`Q++JIzGrZYsR4@EcJpr%TE?4YK9N{Vi)ws}|VFm_t!0wQiS@GG8} zbJ|vBez)_a{4k$`m~1094|<;q#?QK(o6}&5;8@OoV1!U&Q4*tJgkPy@h|P&ZUiQ8h z$>~7+I@*kV^@l2*0yjh0dRvuJ#l&@>RfKTatgXIs5n}eoJvBL1$Za8ff(YH%ff#mg zqOBW$!ZTAOTZwS|O89X_rlRlXnA?rlVS{7Wvf4lbXvln#F6mA8G+xQGtEyu79;^5+ zOJa(|0lWUQL`{3zw?TV<5&Hf`L!mcvCUh%4B;xg~mvFrG<<0d1Z$8Xn!|2#f~vELysA6gm?E-05Kd~hANNcA;+f&aQ~tBTVU#!(7R+Z8%J z^P5BDNQBC0d&tqh`hbVkxnn(gxR(~R=EsYZVUl%65Q8qG*oRUbAsaDW49~l4-=J zRG0~fG-nGS z@pKjJT|7m?PKXqFI2JCd`oP`oJaJt41}r|PU4eMdf=SnUs- zP@@OfBEnLT{@`E#xwp8#Mz}<38jiX%?tv{5FgWxq}}#>@A6L?4mEW z930p4{iLjp2WyrRm;Z>VCsvj!E>NnpM5m6k9iaPb@4B}yCYmXgp)n>Yvign0TaghV&U7iJV>zZC0El6!0c+62E zU}b80U!KOqt2LByot`TBOvJe<3f+6o6j{;-R<)-QTrlsl)6EfUy0WMA1B4XpehVQs z4rEI-4rQC^#|eER^rl+zkOklwW(hF~rx!JXHa5pTxiBwe_Z;Bdg_N8@929z*%Q+i2 z9O=kfYjuYIq7iRpj*duhf6%TOaAVp^gfNZ#{lq>!Y{#)WY-hw_xz|L}Wy6#Tx4aVr zxEQvEgfolZ#&U)B2r<0$uZ8%99^UvZ&xyx-qq-TnsCq#>y*EVJd@UWhh;auEcp&Zx zbxB4`$7yl`0*e<{;I>qM5Ox*c8%A%i(Io>pV_jS#yM-QR1pkw0uZwvj;h^iYdm@W%_mE$Pz99fuFyI zZT!)bX`asm*!ueBeG&cnNZ^H)WQc2}2&p!WM0U-EBmtD8qY zn5;da&8(o7HoWzOvTG79 z4e;W|r_L^#N*=|;2HX|p8CsKH2%t(<%GNSEhjk&*1pi8iy*0elO zMYUNEwlaFJ6(1^6*#Wtgjb(8nB8Oo`$q=1gO44L@>V10_?1Y<0V1c9y{y#4ke!>l~ zONoMXW~o@E=rUZ*w=dr={HR~FsEPDCw#(U6@v#*0Gx_$)ZHGN5Bpq2$0Kq0{u|yhh zIe6uUQPF!B?831Uk~dIN--t%!YywGL*#lU42r@(NJkvb-yH~c{)U>IS1f8$~R_$yb zLlCcpeMGRxD8SzWfK^ z_0wtC>s_qGs7_G|u< znd`Ausi?XxSQNWd+a?tf>T??tYu{0m0=a2t!-{j#uXx;$bx1R5L^XioFQFpZ1PV+4 zE8(7jIg;76c?)7Cfg&)L_*l|De=rgKW3{Z4n)|xG!p`M%q~KBFfzk@XS_{GUi6o>6 z*mTPNlw+M#oMKh(lRpzq6`0_dV?CGI{cJzq_BoL~Ida?i-nyoE=a2c%U4uZ%~W}*yd5Pgfiz*)~)l$ zQIa?=e8so@1uG)hS%K(#pOe^VKv1waMX20>(+jPS@qM5+KAvu7;5Dx9)_aYgFDtAV zBr)UG{qs7l&XLmXD+?~HW;i(Apg)F@P*_d2*fOxxZ40rKvSQ3`N3XyyE71PlH*6wK z&*G30GZwB*ZcUQ(mHyUSg>0LWp9Q;cgV3pwQN=)s2IuI~3W-ie`$mAzMWG5Jz8*6z z;6MTNJbDAKl;b#Ki%SzZ9mU{iQrEPBSYyyNJyAk~Dw6VODeV3bp!F(Uxv6*F7H@hE z1uV4W+(5~DOy28g83c81|3vPIARNox63&T5P7HC*ku3c~J4c(yGOS1?S=j|+)C!m6 zN&QUbNIT!8jSvIg^zfB?rpQj49hN?tl=hrrj&z81RV$}T^^&nQ0QE}GIn)^^sB&xffT(>`2nNn-DO>qdn>b2rI>U3bS0hT@H;I*Rap=ohRM4W39ib7JQg;nNI+vTu{qJkt^NrolH#37lh# zA!`&GRL3?n_#s3eiX~DIWTe<2s8J>g$Pkgg$x#0kdNT!wxUuwqC+W8(4=G|1B^$*2fqoQtpC8LVgdxA*O?G17N=>+rh~+UuzeC z@Q1JPq|{5$Gdsk@guTHu56l0^H)ox#3JwxIS)XW-9lgsztN7F~j32M~a|hQ&TEJ3VK3;D*be+8>!DYOqgpWAq&2xq=RV+OfyPku(Ofo zksr&>sE0pd>+l$Gf2e+Dexl3x7k@O?tUzEy#iL>CoqETj5^i}Mz;K{!H02|S5-7aj z^oN5U-F#6bNAH2hT^FLQZ!Hhul(K8MBO3X`M1N7+$}yqCMakd7z?}Mh|19Y{&4{3+l`Dg2GWX{J*LrOFnP=m{zYJ}g$`sP@&~G6^vOGZ2<>$b1FOqz z|DuS!2Q-SZZYZH1GI6gz?KD9Ni|Cah%(=NWkpV(aiDm^EKB4?6baVd8z_es1UGI-Y zh4xh4oDRpwc!YD6LN_gkCtPbU^llCxs3cd2|@~?I!XFvwx{UI zxX`!ceXnu^2({u}7?de@Hw;sS5Sn5O#2GA)D%m3dIEMg=>Ui_>WhLkLx`>52o?dVr zrFSbUbqnBTySmdS08WAwjzRrITfXT+K*Rcr0%d_eX=qpj&-02je~@~I1@3*z_bLYI zDhAtomGeKHL0k#3wfp%dog1qV?MfhlU-QzLQ+LZxm%HJ5QyLOk?`S23kLZw(80**a z{V{_UQDvJ=k1*mS`Z%jsD|M*a-y0oM6brM^WGh-F7YJF5B4>G|87a--JA~D*yw~AH z?nyrzk8%>nHN35~(3w=y8Qw{F5SGpxY=NReMOX!=U!RwJ-;VzWpvRp z=>_aqJp(xqw&E$}Rn>+qur4Udj2%`!T+Zn`44de>)A47BAaHbpD8ov=Ov zd3s-df#;td#?FZ#EOffq?RhX}_Sse2bm-|{+hvul!)kW?0Fmlrw6@It6dsC@Fk|G3e*|=bOwLV9cuwujeqN6BrJV9-~qAi?lIrAkS%P}@e5VCdB~JyMC1o4vw=!wNL_VXl%l z@CVZ{j4>5$Tut9@P)}5IB@9_pE+GZs`JI}>@qyXOdjMv9 zia-yP6v3MMvp7wn6F-ttv2EVg!?~DK9&_?_b-uk04AtG9#`uS$R{n>fqz+;`elQ@n}gh2mJ}SWSQ27TDUwe-GL4w z)C`N(G;5?>JqT-aH`0nPskYGRcL&GiS>NK7!z-NHe0zEuLa0ZipJx#VjS4*!1Fvv9lJJU96!%Hi z?>It$_;Is-y!rXMRiv=J`O6+ROZ3Vw!~$e3;5t6N4{l3BL!=!bKv^!$rDb0-N&glF zuE62D;u%vAT7Q0(c*eo^!K)FoMyNyRbqZ3Zdgd5=B>%nOOJ)cfa*HW)Tq@|(zTG-NYn+ufy-?II%k_er!ei>T*{F zVQnv@GCqT<{Z94})<@Sr%&E#I{(|=isMxB@htpm*iD>)%E=*%mlLITClA-7$4Z0i& zp|I_wr~<#9Q$I(%YS@F$?Qk7`IY*jWNzk@AqNK4mgS7)nn^SqsZ}Sm0=JA@U-Uq;S z#sz;H!@-ODwd<=mxRg0_xL;w_lNL0oI3PFak1rCC*fRPLnUf2r5KkbbK3u{Ty8{TR zx#r0kR1Llaa;ZHCiP{RZFf@aa1bEZq2EMDN&!T;K`ncX_n1ji7eTu?<+ijB2BY5BO zdAd0|y1E*=kWLXS$htXH8NrKAMGb9p6tdzdR;PCNnfgr9QsaI|mw*)B{0)>ct%mvGsG_^Yx3UpY zy|E02f)h6tTM)@!x0|jx82t-RGMefhX<1qcN4Q76y7IH3o4-BU(0yRPFSY)XcN}hLm#B>s(x=>R%^Y*Ud{oH6T4E zUYO2OnO2M|yhobW(`&^n+O6P3qaE`&n>J=1$BBHac78A6A=%6?bA zJe@lysqWme=Gdk*&%E9f&yA%@1@OMTK^S+0*OR}v*gTqTe{$V(n3qZ<4t14$fOQ?I z`0s$ZI4Hyl{XtN;l-l_dRui?|Z!CCtMfd`iPUN$ci8Z$?pybYu|^>uwN!UmkxE}HTr$)^bumS=7Fk66A_c#PYv;&Q?L5Di|ym- z?R^ceh`T5JxYzBZZ%ky^Bi5{K%#OU^z9-Dm>6=kGe}dpmd|eA1=s1`p9#V)i!QXLR=Ip=V`Vqev*%@{ z`KN#o%&n76m#0s%*3v@xM$tixW+gNx#xWs>8a`_5;JE)CdPqMVTmX5FkUoAr%NIxu zSMwiNw6*@kBKCc_M#MW@U@^&MTW^wNKa*BS6k zxEqtA@#X|U!j%US(bs6=QDO7V>A|foVn`v|r(Nhh@0x`Gtkq3v5psE{_$u&x3v7VM zCO$p{7X(s)x6n(sd3?AoyzCqMmuFtQM_#P^cGy2$O%1EsDa@EYr1K9x5`r9-I%)J7 z&U5x;o5_jFEMA&&i%8^A;k8e<_Y6a$K}pJjj|UnxM|(7^JKbCg@sT- z%h(j<&@iU{2gW_%LXIF3dlO`@XeDG)oHD<|g47Q@`Q1x>R~!2d^26YFPlE=vzoE)} z?LUv|it4ns&u9E-1T37N_Lk%~?~F_czSHdVsHa?;9lcBhVT%$hJ`bHzM~>uU_*{-F zZfY83s%6@~XMvGbg9Ow}N{7SBvQ*C{ zj^|r}spu%Vl^L5ee4cq6^++5pSS_%TIH!qwvyxkzpef+2v{Yr{s2$@C#6>)CaXQQo z!3b5NMKLG>4l$V48#u*iB?c=p;lhZH?NhU~)7_P^tK4be_dWIm&s6uU5Yw*GW%Ko6w!dM)b?N-It#$ zttwe_(82$J#|tYdKS5kI7*u_gZbKYC>opkB@1+{KTFEZOjQ6`19nJ&v0XNr-+ai9_ zJ(W`Y1J*NA8Wj<_Q3E*%7qL+@!^7?|^di9LG1N%G<1wU-+hH_AgVkZwVRIMx?>~F2 z9^qTr<++v@0c#q8?f+H)8|JxM-$gn%&hRX5_+)IGM1#A1E%tZsjAXYR%5*+=|Lh*2z z8bWp&L;oiS?0}_}m_G}5$FYU*+t~XRY2|Op(}OR;Sayy8zy^xgPzA%M2U&>rb@^=s zyt^I&WuAM=X&LQ3OK#IQ4#Ey|OQ_PC)f2?oc)>+;Sr%^(q_3^LP2bg;WKo2k6w zy}>>^ZW2t~D3^3_v}ot-5W$xgPDKehvNHc*nReu#{AHYmT{WP>2HsLPx>}*C@-mR5#5{a_mF7}>W*Ca&Y`0n8XwxBll;`e* zKV$4(=wi`4k*9ltdu;$lVuJzX{6IGXjsII;MQfXkb25*!q}r9vS}~$S4nOYWv7qPJ zm!{)R?ZPOvM8Z)yoKs@_t#95SOnj4Mt$MEJcU%LKe-zmV4IT!B8V7Pnw_~(3Uu!A& zhZCO&c?{FT9IN5lIhGE-HyzZWl2YY`?wrw$T)mWTHgOmiO*MIWiF;RyG^j5ZJ}=HT zU+Zy?6`P!GAu->cIh*=GyppZ7rP>k#144w|GyV>&7#4T_c*d|I;l27n6s2eI$3a?{bHa^w$j>q46HB9-`4O!DvoG)50l}w7G zNgE7&T~%VC?*Lqt*i$A42}18l!dF_UICE7z#0zv7jWv*1iKZP$ML;4{W)5m2 zyD#fyd4PVj@?CShF>3U}N*l7sF)u8;KQj1L{33(* zR>?Z{R*L=(o2(AC-@n#gpsA~+)hLIo;(rp2_U2#8*YuuubnSxdM}9eiEGdX6Gexu! zL+|xn*+Hn%{WvZYaFXneQfeAE+U`Ci_;OUUs_uC465$+;B?Vv(n}yA_N>ZJ}`JP^= zK2`Gcd(Qf-ZAU0Y-^pd0-XhENtAn6w?^OIL4fR_;Scj;u(Blz9_Te4cWe6m1zh#8x zD}NhSwV&^tv7oQarQeRZd_}kH1v;{(dF^Xs(u;^kb`k$J7U&%y7r|wD7hB|T)1hSY z{I)bnvv_dS4`X4&OmB)HTRKX?*-r_f86LDpYbJB7{d0Bc@)ZAn5O$Bzl`T=Du#Y1}#8ns92^$~&8 zH^+elo)ZH~m;rqgMdz7ismE7HwI`$04yuyv&K{{H7%u4Jz?X`GXb&1)O|hn)-9->E&;#FIT& z3;$-S0#>&GvU`>Qivja|>n1GypPobhfcwq+B-PJDKG<6PoB4;qcjdnvzym&>sb^z6 zI1>rt|3+%vMv+}m*v%BtzLH(3$^-8_?R(e2)*)-WF{PS>X*?gDOd7R=Zul=mp$2ck zPQOHX!y1x*QlY-k2iVYV(;}jah<*?RLu$WSE#BfCS|*|#fgX{Z%2rjyFVGx zuv8{qfBi}h=RSLz-xgt1(H{pW;^>6u4id-$Z*}r(%&Uz%kjt;H5fqL{a)8lU2(43U z7-2d>xkYIiV#~9`grg+9gO^-=7`Z@v(YQH`GR^$7?a00C_7^d;lZoDLrhtg59gC)C z8_zRcKH?skjnR-DIJmk@st-~oqHrpbrMDLPudaD=l@BL)*RYqb;WzKl4=?Htzwk)N zKK7;~)L&Vwn_`3Lhy|-473uc;&a1ad(P+-a5J1 zmDUQkziI>f%k&sYb!bwdUF~$6P$Skp?%*n}W4yq;9(p&HArrkmt#h{CCtxU&L3c#F zSfx{|x?J&m2ag1`mF2O&$T8j;r*;uEk#um@L1xmibpZFvc8xC@emm1&2*tl9q7b+< zGR_l~^A3}Xx%Wo=E2F6>O$3Y=?}YHuPRZIykOu*JKSCNJwF-Cm$0sv5{ZSsNddK|! z&EB-R9*MYwz|#9q(}>NM?j^NzALW~B(VZ>1Fm$qdTjGHjN)t|U1oF}xIqoJdO)~|X zY3NcaH6Jzz*J^=-vrkiTTYJ$|=x%%9>9Si%_@**S8e6Jm>0&ThY8mg#W*{Gt>m{K} z^WGvl(5@m(^?1d!U09M}Z}n9fjrUMuBd*0`&d?-qdMZ|biTcOW>{t3STwhdWb9dF=rE-%>6U3ZAzX~ifnk<*`g!7PkxE>a1E^_&HtBEsT-?jNj4uc? zs7p3XW1JF#l{NAVB=nH0R^U_?Plyh-ZI{yp0S;(^UxeEj0R`7qTGigA*p63Lmbs09 zkG=L`$ap0$FOIMG0&I~_^QTUpeL_7v#xLYaEGVRQv}*lgX73vX|?^G zXyy@oY{zJq22Fl`_~&w-xm_2uK=Fnu4cYbC{34W#*y$NKYm0FZd?7qfbT=fV_slL} z=foxgqoz5k77QH<13#N(oZt0XE>JfB2DT~F6LKbY0z8d8z(>GPeBnUpwtx0F#Au|W z%8~$^{w_lpXym(|q<>oZ<7L%=?=uQs$v}KcX5o8Ykmu~fm9~NKPV4m#DcJT`H|J;H z(5R@(rp+A1^YHxZ$0b}RiWD~SSvQ&-PSbEz^=C9s2laZ7kW~xF#FzUr71K8+ zs|u^#13>=hpHkvL{u(2AQJIWf%(%U|B@h{tIb;t=$@;V{b$gv!)66w4`e~*oQAy~# z?E-g00WC=MmbyA!=e--&Ot%xU_5c70)!<^HsM&F_}b+pkyUjDh2OYbUt|k-(+t zmo|&D>UxxEs^5g&fJ~cROxku)Zw9{>#y#^~c_mFkk|xa2B=t8NP#gO6eT9eMkJJ5^ z_%QGIp_>;eG@&zD-G_Iww2yS;V8zN>(v<5T-{RzdZG~A#-DTC?Z37ryEb7|0g0toK z*OeS1dc=F% zZnnO>V|$=obL7D!?s1OMNbP{aKVjp7Yj+X)nefhfJB&r=K!1pw(Pb~CgLA`GSD_bZ zAfFD;m`fYOr)+5?m`pse0zY{2T1M1siN}-tqJO@0>0(QT+j6XoZ2GIe8Hre{zo~}V zes0kAE!zv?hhcRO#x0l#v+`om^Y}@5>Ug^Wo~Mifi^lqW?s&f~$cfSg5d_Rp*=xZn z5MV^sxYIp>T@Z=eo8=tX4x`CgA-%e!XKI@exZ-vEdEg8wHDqOdJIZ4xy^vjl?B;m# zy!YdJ266N9$M%hK^aG5oM=%6@!x_=`HMs+IPFBvULjUhs?!P7;E+CHix`5L#DTTIv zn5s95bc?^)sm`mqY@lsla7w?6p3387_c+9wx0&HbS*J;Uik_BHMKo08ci=&;`edQ+X(gr`)gVe3a=NBIeUPw5rX@q{S~Y zAvZxLzAE>+uWHDlmDP4hDSnCPoNl{~Kbt?9Calu%&HkFXYJBRr3P%)uNg3Y=6NZW# zP;mck!qcK%keG@92BI(Bic0vi_~eI}q-l=_1Bu!T$;9Ka0X8oZM|1^}0k3u=)Zy&D z0>?sY<4C9+0r=z`U9G6SYCcWLiwBy1&6628LpA$%8#y%+2yyox#>~?#SZ;?f?ILpx zwF|gqd=2l$FpBZf$VJmDk-Q<-+sN=%NxiQoZ_DD{)_vZpzEWWECe;dth!$AB+2@e# z`M=GwP2DfV`eI{&2VDKPDD78R9lK{LixCstn_2zEI&^XN6@Nj^{jCKm7r|ZC2LhnH z%Kof!Mhv)gXjmjr#Rbrh92+|RMK|-K?>nBa(|N6NrG-L3Dz42R!^>}NFkU5~!#>V! z(xS1ertB7Wp*}GZ|5?mPFvAO7$)kZ_#9S^XlS1uCj~L?RJL-hseWKqj|07Yg5U$vM~JaX!zA;>VJfDF$3nwxSVs5 zy8mw|6cCxRuH<6}OriS&2DE?yI&)mzU#*>y|5j}u0Cvio8UZtw*cUUbjk*HC!GOW+ zf6=OUcNEUeW=ZkeXeCyR9&>|fPvZ<23{omE1B3TMFJ`g*PY0v{|UIUAAa z&*ZIlIXS%bwuxA%-N-W1(&(hV6shlzLM&012O|sOB!s;Sz$nqV@WT)vPsS8bqs*){ z@c^H^vIpa8>WPy;$jsZSs;B#jH-ClOYfJ@tQJ5aA z4Q`r&Sxqh{Oh~`DesHxe(hr8JkVjGwM9D3bPzXPKXa1C?oz^I6NI5zi91k9I4B?<% zq@;mbQXqluLK!zH(4N#M&-K?(w5bO=1Pa-~b5-~I!#`8>+D6u5_fo>DUL=K02OWoj zxaIviF37Ap27fTns2#JJC7zk1F9)z z|FD5uAc@GXKh_w=%P~Ws$$3}JGRFnEpO^KwUYWn?6jVBl1e%!uFfuqrMYyV8$xuu=`_QH~2#JWz zi`;M|`Cs0PL|LDrg4UD`v#PpVlb$vBohJv0n|Kf-Y*VlGUm0^r(4NY#AsHPhR3vmA z@EUjA1wX8H#|XCbcSX|D(h+aGI1DMrOg?3zc(FVAYopey*&Ft-&2loJTTWdpZy&_H zxKO8EIVS!w_s2JH3O+_9f_Cybo*}GXI!TwqC^vWVOJ3%6PoaeJtL&yQ}tjqsrNLB7sNd)1Bx5%6-G)4`58CfZ?H_LO~B z(gt1ssA}@OqWzAHU0LyEceYI=Su-%`r|NWZ0m3=&jqkpeOZkFwYxza(c%QsEzY1N^ zWXmxjE=np~H_4p6nY;KGF?qf{ti13fTaqu7xsp)MK??-xz`}KBdW;C0y_9*_WxZ&RWA?$k?g9{2B`v*}zOjCmJc#7F1!J z4g6sZ76X$;C#Qh@gZL{9+)mzA7;&Z!LO1`Et|X5Tn{ksJeT=$f`@NgtLfuu_xJC(s z+WjtWgUuM5678;Ev3%FJSo(dE%aNL3iec;m-m9sPoTh`SjMrynI#XD%Gl_Ys4XX47 z$94b$Eao9eXT84DtK`t!x~nkn4(>)dG*>pJ+sUJETNRq-hzQ>LJh?x+x#*GRBkCb_ z^=IHykDfhsr_IzS#OBgihi>Ga*Y+=j<(=U!ZNy2elrCV-O~M zf1<(AGgj#xX3sE%@En$B(+W)nMD&i)3)8L1MVVWdwgGn2Ybs9Z#G zgl{1l=GaB8^X1;)-%B-U5YR zz2kam?@ z+2d6)lsOvDw9=7F>D7(z_xCBF6zSV`Ik&4tqo1orcSN~ORhS{un;z>v_GKr*(vLQs zZuTYRM!&(*GEa&M_Sbj`PuCDj;04%SEHgx!j1YQ5O;I|B%I=g!|HbudXB4yfZq; ze_bb|><+*hIC#yFCwHhS_*IgrDRIqEna!+B0(63p0R1pBvjWkd76DE+0;WT ze8@WCzoxHuSCWo^lEI?NaK(gG(JD`fGFX#brE{Z#@1OBxo4stc_SRthb*sq^t8q6; zRGgJ#DYJ-62%Du1)HcV=WZ%axv)Pb)>HG72x%>#71}O+i-WlhQ!gvE71DTwj#y>z1~= znnrQ9jkpJRKvB#{ObVfZ!+P*wmH?eF7D>ksP1Ph01iwSG7*hW~GO_)`e@RuP`LBac z9AQRcnVrD0sndn!QX2e!goK8mCnTl?j#^3add2!T261*xYB$C-N`0kYZfTg1di^oBSSkT*MkLEi>M|=->$q9P&2bWR_ttNMJM#M11%_t@F{g{ z)m+wY<&Qb{ue+l?ioh*Oxhlx^J2 zD#>L!_luis?yxHtuk6O7)sSnEV0e`S@au9H$seReWhl52RYT=W@x z^_yE5pj!IE_Qw(mVHIvKa?wIORV1^jTI&BCpbe8y$^QINi^u|n%x?}+N^O;gWhBIUtiBM`gNHBR0!!j-{T3g1pmk)#E+_rmm>CdPjqhh}= z^zF_eC1(LC^es*{gKLzHUTm4AOm2gz&^G+ z-btI?*b3eHaD1_+Q=u*TQD&*vb0Uw;u5SZA+DfNZ{4DsFm8OaFA(^9n*nC8j}!I}4v{^$h2|*Ey4XE)*x7w%bq3xCz!~ z0=?5C>LhA%I)vVrqtjcd@tX#8MX}!Bu{Pr0dIz1LuF2UlJnUQkzc>M_74FW^$h1p_ zaVCYtVc~76wIjsxYZ=~`7M$XK0tS;XN$xePE*|j0VEjd|9 z@$GqabMfI(h>Z(aTo)547y6b$398))RrvF=46+m?u_pBg8#u=|k-<;V<>LpMSZu%A zWu;;fu&82V>SKUCE2URM4Tqn<+uRb_)@q3Z3!+Q!1eMT4tpNs1@1o`_Kj zE_fG~;FD*2xtkzi*fNn(^RxPB2K}`QqzGt;!&O5oyGwhYdpd{I41#R0?z0OlEmB@l%xYi7F3%wb1GF%l$Ug_U<*L=GBw7zxhaq_dgs|8fU2`-_Jnv zta_Zi=t!(~&gB^~2YYjso_BYjyS|&GEBt{CU79K0Z2ynV==v`Ab}QV6900+!_?$pyUo;!Fu}~5x;HP`-hZg^VhOKyv{B!cz zSLnU|3SMHLZ&5(W_SPeWIfueR&7MMN-pgm{v8f?fQkNa&TuSzwFM!)UfcH(;#7&0j z?VwM$oBQj-7yt8-@)MiE<{%o97lBb)jB?aXVp<>kwjjw%Ha;jtYK@P-I(aQZ(%EIt zm&UFAO;K2RN&m8i$eO#e2%m}uel%|^;t=ESn@o;`X(NA^yn2y>K`f(o>lT&L%GgkvkT}o(2b<18=O&Nt@Icwy7g&>0 zxJMK4&wXu?p~Ul=QpO>kH6n7c=w^_LaMo#0I>bBHJ@dsIh&WWONEhvc*m>IkJ+OW! zyct+SZ|txg16`m^Eh}Fhc0)z(l&3;36)z#D^pv?<>V8DRw5Lj!APNP_6VPKMWo}t) zNk1Ravs)9;il5OYb&@wuH;5$MczoL1`$J#i`}b^Oz;|*NzlZ1j{S^IWAm#UOKt-6( z%U;GsQx7BGm%}SD=NDytLP0@IPgwePpu}kE@&AsP#_|D(Y3g=>mox<9 zU=NI$hOMl*{GBqJzIYNt1H!LAl3oYvnSMXn3(798LLSjz*{mBEqFmAe)w)+K-(xOV z$S{6VSmUGCxX~=|xMXD9y$?~JH>pdrxxW229lw4H2>{7h!!EoGzo~ZEi+J#L4m5+1 zVA{{Fe3WrF&50)yj>DEL<2J_ZdKCw(dU3Oz->b;3rTCT=i9(z*8B1miY(q*fBZ{TT z%It6zs$!I{Q>aN~^C$5P#Co9_bV#KZ^|OY%f}9704Vz=nzi z50Lh27xk4Sg67CUZ)$^h(VX2lO8Cs^oMRK*4M}?b_~qb^A~WHHMZPs8=`yFVT8*Uk zKkQUrj9MB>(MbTx2!j@s5|WuwR@+!dYN2yd0u8l(K=sxObTOk22=)YCRoW#ijx%af z-%?2Q_%^2>hejHse2EXc{afta?q-#??6(wlX2Z18O8nzklYfmVb&*JDJ@))R@RWTK zy6RLxHzjH_e)61rH0VCSr>b=(b~w1Eca3XZv=y$jny#45%@@h|F(kL8aP*a{<@F6k zo3fIHv7M+KwP)@l-lHKc6-~R_jJZMl1JlQ7(8#iFz>=KjXjG)?fXey?FYz*TXLzR*~Q{=`a3PZ)NtQzII`Y(@gal0L^ASPd?aXJ_{skJnt zs200CDH&^B)z(RUGE8aAQXs=Blj|GOOtn|;3{4h^7XM@GJ`kRh{Fir)EPE*>PWBZU z&JfCvcmeNz)Dw1s3uC($?v9-=b)_Taub=7XaYy!Y?x;Rj$BBIFU|0vQyNku~C zguU$3o!YEz#iR4bJ1~#7I0~MEmP0oYyJ8jam zX566VsXU@W2kF6X!YnB;>mu{c?(Wx+(w`1_o(_6k4D|?gyZHqi3Fbnl!x0nPX*N@A z@n&8X&gP@e2CY2A;X0zY-bV^&nc9=HJMK`ZJ$G6a+ib4CTk!KN=pXIqS)^w-o@x6i zNq*}qAmRYi}A( z<`-^|cKM5Sr?bk87;61ylGCK?uRp%812=yW)>3N3xk3%@Ovz8k`Ei+uaz7 zF%1IpOK0gu?7+H0&DEsV}{U(9s-YbPPir5~;|whpU% z>Y!-EwU32NEh6N?*vrKSq-HZ|S_#Nj9QEkdO2lZz`_;L$XgZm89JyScnyo{>mYObK zJ@Ob+Mg|pt>^}}}3g|U)D5H9;U7SDK?WicL-5i+Zz9ZajZJ(m0qROuSUrm||juDPEzKr$I`0ll!l~cH_^}lk`k?|3Jj~roNq{Lom*kLYeMmmP6UKvY^SGE5GCQVNw zY`sMt{e4%M=M5Kw^He7oOFhKB_Ds+cc2+$^X~4;3^JJt3EB@njTs}49+rJN#4d|Jt$2VDG8&&k)JLPp+(!pchR5; zD!kW0gg_UutwbU{hCVkSY6DmLEd_)e6jfNRx=mzEY5WEwJ%QAC{>o;j_ByQBR;Xx( zic=`1sZo**iFt3Sp;8bivvwjY8Kx`t*y?uXbZx~Wv|wVVK33h&01~=X1v#p z=y2|G53XujoL}GMrE57M9|KQLH`K%{iGwn{|wG#~va>Fm4^sg1) zCpae+dS@&nP@snh=_Nf&&A@bYlFNmP3+_EJDlFoM=kX4C7N-869~^Y6t1UPM_i%s) z<0lT@u^p`GM{srxaISUzaW9el9c`+_!GA{VDz2{;aF1nh<59iP+0O&DN@s?>xn{SS zdac{(Q_c#KNW&ZwhTFv;T7T#8OZ{;Szh1$`@|Iy#Fi4EHPxZEswYH}acOywT>Anny zuP5aljlnqC$D#RgGu6boSoL-ZDn4GC&*3S_)Y*Oha`kiV?12_PM?U95?h3Wpyt|3T z<}+oQ$pVv`BWUuMXI5HHpHKL6buo6>Tej?hrcdarAYup{r6aimir81u8}WE{UeDWk z(Z%V*I%_Wy@lk?W)TP=N8SzJ-NZQhr=x`#6EwhKR@v=9V;rofddW03mdvU6;IWQRM9 z%WG!axYZ(Q;_ZtQ%Z<1ovsAef)(l$@0g*w|4Xhe_lPk=9r=k@v#TnON zx`@V?#UZ9fnY8-ytFWTiF!8582~m0~XeM;&KdRg*H?`VB?UL1Jk}#&ol(RanJCrb~ zNW`sIcW{%8K)hXIEQcM_+>PbFix57Oi-#XCVxXv~U7sPGbv_$RGNm!q4GrkkA%=+#PyE@9F5R4C*n?$i}@OIlyBsFV%>4TvoOcdx4Hwqb0(GE3)`#~J z3A4zxBg zm9TXm>W1?AfZF^?9W*p+DHs=r4LmA*d-MHkBQhNCCv>D%*V)T;kivW91v@(Zb1e_! zlcS|psrKBPRsfyeH3d`G};A}>_zPT%oo*Z{&8XAfyv9Fn$dP!U*(Mmwhd zUwv%-zyJBsCdA{L4af(4{l8QH@zUw%%USjRaiITs(ig4)#A91PlI_RiznRK_)mMOQ zTfpLf=KnNf0_>@dy&gcAHtH5t0$gdv`oEdPlRN*D19(997j$ER4jgG%;WrWbuYxfy z=uJm+m~Qb8Wu;!%F1HY4`i| znQYy35+fEzW7mX;^W-oe8u8)xQ|y{^C8wi-wm1}Ij7MYNSI)G*tqLZ{hAqt&BDasFqNA_#hF^xvH8;#X(rqnj;+p^ z=`WCsT$}@H3W``fsDZx%CR_6uzQnsVfV=PAWAC%G$GbC5r?*cecq?t~4kl~*lO5?! zWRRk{x7-9r4wvnha)C(J8PukqM3C~3{4H&98t;9WK=Z)%`xMa}cpeA3RnUjCoW(1z z>$#ni=ORt0!3yFa`Eer;mm+&6*W{(hVbh;lV7NLUxY}=2HAW3 zr}+&0H)Ejh=kw`Bf$k=}E*>r}z?g1@+E&xc_~!ra`5Ybp9R7Tq*!y$%STZUT6Vv@S z-%$geVelv6&zn7(AHSk}dN{iws^CsKSy-cxsfpd5jqSKU=ovp5Wd0cFW(`q9r@X4# zhtuLXU0`V*2(PfB$)}I3pUiHq z;Yb&G2tN=7&hOuf70ZNy20N-uF){9Qw8=t>UkU%E-LZ&jE7B#6>5@*-H&qc#nbre> zt{=I*2g6XaHP_gufyH#5i@Dms9c|~-u?=rdoMQlY-j}|+6_9kJPS}V{xP+@Q{w=)2 zP1kGCImfu)kNGS_Syf%ta?vH7p#Z_AI`k~E)wzqNL1C;M7fsy1^xj2S z-YsP~5Spv_H3GKs0g;A@1{3eZLH;whBai;TST*sV?kWOD*wY%^#4k!~`(GRru>=!L z19@nMrfSN=LilOP5f$()7uDEVW>hJNBJ)sQ-Vcj?;)uqVM*Wb}=dIeK?QlPmV|WvKKi73YcXEfC^hbU(Z7 z&N7X#A!t~EjiDGDGm#%~uI+w*Yd*^!1hSj%B8co(5Satr7#!+|$asZJ!`^x-e4ALC zo5=hB8F7c?#!<%3l7kNlXf;>>Hn9x71Nx5YlXN!|U#@O^^=x&-!T{_Y%Qg87q!GPT z*|H|L>n?Rv%(=3o><5QmZj9C%MyO9i4MLrhFUnL+qopTiG=P*tC+AG{Wm(R;aeb@L zYi1%A2iR))IZ}Pj3t3x0Ph>WH$D!+bQOy5pTGkd$_K?KR=k>njF;PL> z`l?2>BQ4`}Q*U8!T`gVS!%9qpmW#_VeRdX28%G98PBi^a3eiPv;I@y}(e+C_Z zpf-NP8%3V(qN{ALQZtDFL(ij%bJQ%Fc zZ39F$wcS@8-Gck>gb*W$ARERWvRlfHonnc9KCH#Pfki=k+~}?PY<30$>NLyNc->AH zm3_ButuA{X3*4;ox|=P#Jh_rne0RMq*Of2wbh&BIQEsW#iD>%v#U9?HmuCvb9DDT#5ON?LuTIw24k0nIC+*8gqDOuyx3@C{ z?s$9qR3a&p&DwpMawHeoXs$WfQqdH;{#iyhc0fdGOp#9|7Jc@R=H`vhPL zu01_5PIQihoIjsKl-L?MAH9-j__gzn8r?#PNW)1X~`Pk9`AqX`uPBi0|7%m>xvy{;tFfYwUf=ofZ)`+q@ohzw?F@p+E)A<{nLN|P)}M#|Z!?%WMD|6dO%*KW zabEu8dOsq`G;;fCj^37q8--=w?=2u~{O=ri16KSu{)S`+bIY897z1`O zU>4Z>guX;W1ypSzbJO6Ra&l3)dXH zrF%cKXVUX&RO3mYW+Hhyv}=AA3%%f67*zRUycmF5VX&)&RJW^Le9G83PRY$XqvsK{TOnOY(Y7Tz! zkV)u9^gszcslhSQpj(G9)I3J&LG&mG6hfAtRzn21S3r8@?l+ey z0zE)YtKsN5yLzr|h}c$|FZ$KI;!}?99z-$miw><(vAD2lQkWKm)CbiLuNf^-gvyG3 zLCJTXyq!I}-dp2qjf8Cio@K_7+xpAK(;#T>tLUDMNM7);R%tsSGLHiB~LnV)WwwV^;kquZ`h3?EV&2!Kn#w7s(Cuva(m!Ys0uzUQ3TUH(UZET zyxAS@rvcG5lA5pePC6w=b*ro;g9X3B)e=?IKJ5T}V^>~DW6zah)yE1`+PyJKzmQ9iW_p3aJsHwH?!OsQYsxF()R2Sm03xEIcVu zRW`Ot3jdkDNakM70f5dYfCJrB^SxHr;~aT+?9+*QI^inj_&1aX5TK6#na_8TT);pQ zFc^|D0{GLHx_?(yzq|q#{Uc0Q)&WTL=as6QzCM7P{&!CRRBd@W`v!b50pg4Qw=T`J zH2mV+_;C|gTP90ukN-dFkHP)7g~Mg-dlmU8DjHrCC*MD zSczhcLVtXEmgFqZ-b=&VM3wAnUJaVbDX-AK;LTr4l`=_X=&W=;SX+X`G1kV|E-~gi zRsO0bK)*j|6YqKD+lQbeu45wsi=zW6!)mT5dIbz(aW%@&0r6lOa=k&o;}!-A8G437u?iO-nVe-iv<2RiJ!{Ss^7Wt) z=A?ws(R8y&vzjVwf4?U9z7eNZ(v1TqOIrKWutAN3XrE$!gq!)|H;Iw=#K~!n(>yMz zvOjEJK-|1>khYBU)k!mVTYZ~274_?&cJCcu7eR+lTnDOrPM`8`s4|7MP`JsP6q^ik z<4>ze)MNG_7tDauBhVByUrbrOg0G(x_tUm-VSRcb<;Vm-;7-?b%GgxDd3Gn_uazNa zEuLiSOZTXfb}$ZkdMUB$YCzNk@C z!W3FGRd*2}_K6N*JmU%Yy;Y?rn(qBk$J&HiA2XkYfhsIU#hUaDEQ2xb{7lopG{@t` z$w-<~!F>KIN1l5acTV%7JQtPvK8uxpj4LQJ4}osmy<2t#KN=x^*WPf7ImH(en}uI~ zv|JYRH=3OLSP?hH{(BIdj3IXUvE9@UcCBssC?Xx&V44PDzjB>{d5zNpd@j_8TNI)O z{#h`7(gspz*VwiwJ1?Vt%@sqm(pc-YgYgXgS&^hF9;4FjCI+LWV8dLM9{%tTcc-Y@ zZT!=|N*~I;v>ZMgnfEm2opO|?N4aVP_yDu<>C)xWL@B3Md?8BwcD!W9Gdn|4CM{iS zqEj}oj)c9F#XY*M?5gI4{;^gNa5YZ@2Mc?XPH|2gp5n&=1K;Yz&tuo?w46L_&$N_@ zy3U!5?>>zmar0T)C=A=2GBE0YdHd`E(+iLgzYcqz1eM85rn7{59Dc8`N551+%A3`QW}jH&|( zH3-*g;s4SmJ^No|*m8CoVsKlj2-?`vg8aO_m6H3_aIiu1F zbg9TJu$5ee?=EL%m>!WYG0S`>lIw@-T*$BEAoJ;phEK6!=KT? z2%?9)6E|aKvBsS-T9)nu4m4?=MqL7R`Ig7(J?A_h7zoo zb;kge=wZjjm}r;;aMhFdvzmD8XLCeoDI0)AON!v`X{nA5a88i`Flys}+aN3MBn zanWmhpicDuf`B4u3BBN(ds$kMhokgmGn&ar*jN1N8_2vY@p_i6t*EYNVAlPtl3L zJK_$^Kilqo@iO0IW0+Yda}R3o@SJkagrdf#x{#WSym~)X z?)KWr^!Bp!CL4N9prI#yaQJwV0v1}jz2K&4M(1HQxQd+`rCh*L@Z^{IF0h|vXbu)y zz&0HOi^6D|Sf;2$BH<^j_~qgO{*&jz`lyKM%nn@+r#)W>u(1Uu54StF9uv_EDErmW zGRxN_bo32}izN6~R7BS}Ov)5$=OKm}$ozxl@ys9=Vy9AxgMegI67d^AvunER6RX z_z8qR%d!H~m=C0*^q`<#Cas^2?8r1wNP-EAD96Ga(h9NUX0#UpbuwXRUUTo(K#@lgVSCH@ef3i;@44Fxrb@fKAd_)7%P6LZK312qIlL6KRg|C&_nk* z=BpGYDkfw{M-F?;MSq=R+B~Ukbe>k*gvTZ}6nmFeZTVCe>28_k*GG3(U2;DeeT&!u z7m#mX6+zoLHP9hQUdct3qm`&+g4Rsl#?O-x|8PGDsiZ~X9s!Gwh@>JDLKu^bTg92? z)M*hSczm8}LMso`;}D8?nA~S;BS7OW10@B#m}e}J(5rh+`vn}|e4Xpm{?$NYV9h}R zqwceZ&mo+7_^SwppfK65`3wuY_1;9yoiaR?PBK0IVAl@Y^HSdI=bk6q%Ki^J{_X&u zM_LBJF+2exnKmSuhX&!8U+8sCN^zvt z;?#M51TptDM1`?g)|_EF@LgSn0hEqI`_8-K40$4Y0Xfu|R-EJZX8%x|B;g zt_Xooi0va(9pWowGo9Hw~#VIfODiqT4_7Cr$NA6GDqz^C5>)|I*58p&Wsdtdz<{+f}3u^u&hF$t_{ltefK-8`}}|?P4}< zw}||+1eRNyO+&xBk9}%(o`&@7;QwRq-JjdIu|4tM?_Yt^+$u>KlCqPzb7#D>lTvIs zqfP8+Ejg+E#|GquQu5&dPNE4ZQ)H}EEq!Y6qw^GltFVbIo9fxeu4Zz`xXAU zEwu#edgaWE6Rp*bym9*Sk@ByHzR(i8SkrcNvsb*%1j^n?@$% zNdr}Nr2!C=cnhfGrs{~lSD1u+SvVUsWD*&%-r1x&n|A=mLAV3~4yHc$Qm5mn*KO42 zG#ctM>KM&1PZ9FD?G=Dd=!-DOiy)|rj22p%dqMv9$%jAwHa`9O4# z-6s-!j3oHZ%0w$fXj~#)L}(dm8x=zAnrWRYB2!)zM&zeH6hAL&0YBzYK)MzMPlX3F zUwNBbI`tx~q?TWJnk6h{0!=7mD@cLCVN#9H7--q2h_ToV;WId09be|G&R5Mr+_ZfzNV1YwayBm$nrjNk zF)^HcG^Z$4zp3BS-xjAI$w|tSgj=7T{cp-eSwPfy*;r%AJLS)QvxoC^Q61FN)cd#c zKUb$;pRS+PE}c)71-r`GjDOlh?bG@jG(j?GqrT=jn{x$~R{%m3!(DTQD{n>lQn6W? zG%Y)ynMk7xV?}-c?t65UXp!~P5`BgDIR|?WbOB>gCJ79WP}_!@iXvK5^xNl(sDdYn ziD;dcg+o=hAqthg*3dLKt_bzY(D>^!A+Mz-8mbRJFq~4MVT*h;W_q04FvEdjQO=q! zvH-;$e~RcbHr3C4C@Mfbgn;QFO4Irw_AN0pFRzzpMJ>$%*|5?3d zyL`v~yPYNETJ^vHYgveA0EBir}t`kY))dJL5s3)|d@6M}Rzp3B$W`s4ULhP=$k*Y+sJ{2x|rlDMQ zYfAx&L3jD-@crs6IqisigPPL~)u!8DZmTn$yaD$zX}?@WE0%QD-B0iL6#Lla0VF&a!l=DJ;D*9B@&>!qNvK6nS? zyOya7h}SbQ<^|*xnxd65gH%`-0Q%IbC zzQUd|mfNz*Q*H#8AKo3C=!N`#oZpYf<8kX`FAZqlsJ7lN4G7~1R#si}w5neeyv5Q? zLkKb&Bn6~-Knmq1-zAUI(MrJ*_FEm2hwBRNBAhVYD*Rkb%xCuJMZ^(oUWu}_)A z(kfu7zLN38cUi!e!cC-)>asm4s&*8O9FMtVc6MY+bw_UKCo2Ten#*x%C=03-3u)>f zYqrj^1xr~@lX~|C8|sD8bw62|!O|eckEgaz$k}#ploOPs?5I?oV4D0>nEAot-K$@C^2*a>Gny=0W({ptb z9m5B@Co;FG+Cn0$Gvn9%-^v3nr1`TyCSBaDJ*D3jb>Ehoz26x(^W%MXRG$5_*~H%~ zuT$)MF3L{`zPVKd4VV!xpf$uW{A9BejK;jKBXx6%o{UC|C+tw^aw=k(u8l!&5o^b~ ztu}5bHMR+-Ei}3XHMS{uD>VAh)EHoOYtEOmPd#rAEUsL+4Mh_yMCq>T-d#qXu$men zkt;ITO!!WswUoNS>mNVh*aw)*Zj80A8avp@9&YSq>m|50c70#g#}4<@Lyf)1K&4m| z$u!%6p&o57(AtwxzPrsvqr4Z`NfB<=H=#eYXcSF6Mx$F(Nrju$qVQAl;%u zaIE}4CA;?(D=uxb>jTwH+g zSqrohF5Ir&a=(fNK4KKFG#qq?pleL7W z34#NF->4K0KCY!7+sa8By4x@Oakk-F_bA4WK6|p|SN8-?QmL%C4#hCpIvA&WlAkokTMAV#LRa&()cs z;jy05Dd7SNeykQ^>Bz3L11FIiIjpq_!!*~#`qY5W!ESA(DXs5Bb1JQUYP-=gDz4VR zNA#N=3$QwOv^{J!F{83;scaXt^^dAKAIqH2YE`#9C@JfiJBCWh=BJ(UtkrX#BzSdP zLB&wd_5wx$2m^4T#!k?zjj@Z->*s_!OktCSq!%4prX=N`bpL*P-WW7v5>kwtzwQL2 zysT6gKneDn@M_Ts_~I=q?qz<9bN0>Z1_`N;Ji6hooqucqy?0Uw_{vOsnjzckXg9ty zvf@7jO=LGdG}7!xYltM?+>=HQ;SwcV+f+NgG&lz5Q6x*z3aZc_Aimm46Su6tWi7E4 zAvH97Jo6Bs(I2sMU~M;a4(d3c3xG4?Gw2255u0HwG-1+$ z#V=r=j-i`lz}R9m#&|jW0u$$$=w1HJGt_`A=T-^gNt*1-fr~#?3xqsyy%gIL< zQuq;$!^gX-#MBK-vxF9`@sN_?IhBU0&P+X{vp?7f#0}Mol*POW>U7wWS1pm80X~=( z;_pF4T4oG^ukbjS*R0q_qg&LX0&tg6V2G=X%A+IWUq~6P+;7&N@5S6>r$JsHVQL;4 zpso|s#3_+EdBf740dBXO-J40mq@WfhpxfNXQM{a#bvvTBTxx zXVH<81Li{0B3JI{_XLO&>%Zl{nrf=x!PdeY*gYb+1a+$^?05kE+<5o}r7ieb`*k(A zd#myI`Q~`7Pg!x#SXaFnU|nvVs{0UBcAIyq%2Y3x>E8hUX!U4Vz64o6a)KxIGiSpr z-7=QKcm*EvoIE-pgAJh!dJK&2;@RrHw2f?kVLJFEEazoVaRj;j3!o-H2Lk=}+|>E{ zkG5uB%baKh+X0WN5eC3&jtpJ{Z1DMC;Z|-&UjCK|mh)(3c0H)e_-sg>Za2p6cAn!P z#%o_~cwIM07Q48+fAVG2LYllGLiMVPcuW&17`giKC6qhP4E&Q`CR9i3kQri?fU6&0jtHNyiT%qpg1F3jZgZ@f1yt1CG&rhY_`ag) zxBBsKdSZ}_J^q`Q3jX(xXhQ91{Nu{|zDnzA`hB!GTewK-V`ps+0=V@>zMNpi%as7{ zoM|RvU2wK2v?gL&Fac>|o%KVAHGVuRL92k*8N|H!1@2OmLE(rU?6`W{X zQrHh_AQCMAitcYUovx!*nCKTm7GF4O_S}x34Bts^3(NpCFU0h8)<`oOHC>XFE3E-c zG%G}Ab?4O_DAu1rA^|0OK?fEMiL0e2In% zZsCs&dOxD}An4(Y zSCtzw?xS{G{rIw{hdLTBFEr(uU^v)96YF9u^}6%QnDH;f4~@&7VqFBC*oKdWRC?;H zucRN<>A53=7aH^!6&z-xizrvr{ol@WtsPB^r8Dd41Z)|R` z-gXNk!Z?dh@R zMFpBJzCcn}WusoaZ9)6$k>RtUfLj$&w_@xca6n@i(D-?8*F80{t0%CUp8s;6Y=qC? zhztMGToOgqee8bKdKFXPOXYaXmxOW6%;*I6%=%^WWlzMBA?dWbmJ7FFm! z+}ub`0TS%q34oZ2E}9}0ntKd6icrfMd;(&LOZ3Vk15a56<;WHx^+?5yWaTO?r1#c4 zHVc`)z9nlv3Tqwj;@S@<6`a>#* z3}*)VIlsEZh-0H2Vpy)d-yN(lf-*P>J!pXmF0$W4Lrueqs=wo2A!irvn={=4jw`g+ z*Y&)G1<&JgMj_a5nK3o0jj0c5v;GXaGzerRY_NT>6xKj(yc_ks2waJ3$M!M1{tEK; zSq}zXaj-wS+4UDpwa*xKIpDP(+lN2MUKt;snxv7^Y%@XQ9qaJaxYxIkH8k_As>(yy zCJF|A$zVw`nU#sfJ8(ck#R6IyVN{opVhpDFfB~2$0GAJBTLU3u)PCR)-Z;F*E0#k} zL~BBa{{W-H*1(U~qE?t!2He`Us^Jyuv;qUi>A^n;T|F4aE%$8g#iQ^ zrm(&Y#0(?}i*4^_!z9Q}F%~2U=*bhfLKb2b_9tgqLXA~uR*DF-Pk6Dkg3_9h3uXbt z-w6@WW*A51srZgCMwl4V5b$R}kM-3vrP1v@%^ND0tUr-VGg|N|*Ap0_l;eve;<%^> zVGS0z{JETXOS2}3ojL)`tI?)Hc$NvnuNMc>>nMdaYNu#;*JsOm5b<^fslcpm@UdtX z+BRAqZ$->%*C%hM=E)|(W4%F-DFiy_j1it{^?{1QLF-0u-a(H8j`V4}XGx;}+rZDX#JS8eV>eGvH<_0$J_pc!rd^YPhefjh9u|w~;-$1YjDBkuK7;g?p6$iB zh34(F@0f3c#>NHitUYN#TXpgcE9fbCZ;IYQvkc5JPEpwh<7>Mv|LAfWd7uLXmsc(5V-mbKOcVm8yDm2 zhcH$zZ0?zY(m9d5VHIZsL;aa9harD!J1`(7bL3}oLYZ23RPWwHoJiLRTysd zIczDb)n||$xv!Ai>hpAD{%akXJr=TCeIBH+-Rg6ILU*grXT&WAysTIPDOUr9Mf)>T z(hGnde<@kM#B8{(0QZ%lN=-A0$1DdDGX9cX+ik#2a^dRe1@gk$lvkUmS2Kep2&5#p zc)mD_;cxyYTJj7v8NGcKdVqtYugJ#C2msrFB${T5>*l`kfpb-5%q6#MIWpm*MDH}U zF9#0-Bdd`S-ZFG+VEds*ei)76bIV#PdNe<^71pCPHV0fk{ejJ{8qlamVioXe=%mST zvkX3Vmo}dM00Q&-^Houxn`32AI;DV8Cy@UR2>7#s1b~l)oy6`?%e)jL@>Ul5KNk@6 z4&?PpGWAv#@X6lS8>e!p)i&-MQx`wa=@K(~Aw$-zstwt4JHr*Z6h!7`sRniEa1NB| z&f1|XZ)8({W;bMB+l(D`ZMp?Z6@CLYH*j~`P;$ta%4xyoLB96tNm)K;TFPz5t!sJ) zF?2J(O~`Fnh;8+36>MvtaJw5mW6cMG-?j7;_=AC(+wg_kddWam`5sa(_OCAvrl!r< zkT)k3AyNtz+5WGV6yUV}Kax?Ns|h(Lii>%|YTp@)xYtZ@$KaIG6nIAyhUryQ5QZ8D zBw4S>h9h#nkP5{P7aVkS%a%u@0av-t){ZX4(a1$+)R>bKp1|9D1m7Jsi<3hH#eHcU z+XclLQ{~HqqEBESM())<8nK4FcPg)oMJGMHoi@?~2}r&ea9FYFb60YAuWR1gnU@0? z8z@j%;~$%T;H5lr6}}C1j2G}ZU0;$o=Xa1LT;&4D+$2q|ML_Dmz3Rlq01?o4Plk<(t`&8=eTZjpcKGeOl2ty z=~$qQZ5Kh$$|mo+jt$9qX)jm+%p~51&8-$j0uJ6BmvXvnK8IC_GU#W7tKGaDX80j& z7UTFRxiSS7R6z>{p=mjrC77ORRsaiwnrtM`Gd993Yg@SyD)QsR0tOJiXj?&*j4Qq{ z1~$!tY<{aTQ_artW;VEuPkitV@kYjbyb^Tl4k^2aCRIg`Z+!ncxarqCVQQ%g41?t< z@*fQLUBQ<}ZUnf#yI;!WOP%-nJUOVU0NJ8>I&R~nbqM_ z*8{<;c>gwug{!K?71J(Fq?khuUiTi0o?xynz?BNgtsq*B zdf;KTcx+%@O5wHLZ|H2e)aJafA!dnbC2$e!z6 z;bO)NCY?j`VIoE#W4xxg7T{Gm1=pzc@b}p2U%EqV@LPJ2qhHdLB`;|gF8JPQ0b+c| zdNa48hJKKe8bw?f7^ycF*x6Z19Mp5p5_X4(IUTD*g=^~_>r3kna~gRdDBbt{CYHJ2 z3wW4NcJ2YUpdX+#FgsB6?3OL9Hb^tjL>;z}f262|_38CYgZ|@E{R=4*g=LVXfjP`s zk#ly(2+gOw$SL3~;?9Eo%M}Rf=?Xm8fcL0dtjW6z79{gI)wdnI7tw;vWzK)?Bdik@ z$;@mu2r;K_KLN4`jJZy}Q5f_Mh@f!&SGsqfHjh03Ci%`uCF!?d2U1CY#=x*`M5CQ? zn3aP6QnIS}iwifOkV{W*tK&F1$zeojQY>US@5>5>?AI!!DWxXsicovWf}sfRSInuw zM6pVoqnYO0IoAqvKty&;exA7W2`m@KlO+goA~tbRI5Q~-k6D_@0(3Hy zJix3UmTf#R(|AbM>433@5w>0hrnn$BtFw& zu$h8o_yb}x4aQ@704%0Qio^7z`8;VpPnyqwu?Auw4L<|>!#;Y#J@TIi#60?(mFz%l zqhWNvcTA&4foJqkSVkM;7#$kJC|n2$#V*=3sV*3RWjOfCVBnQU4Z5;5;L7j|@(|cW z4~H}|2xcx6kLc+b{Qi!?f$)YN9Bb&nI76So$qy}Y7lbRc0j5xYF3WH{q42AJIF8Uz z3?a7+0p!w;9~7GK9SkC|N$&oGqXBJ)1T+){XaE9GIQq{JvcP9J^3Px(hoOi+o8r$L z1nFnLC_jVOb{M+PrYn0;RG$Yz^f^45&jv_715tc7K=2uk-m^6f%@EX{P{f`=XgwQX zm>i&j=pb~SO|_>%6f&HBJO~=kgCOxd9172_W%A8Kl!l@145OyQpzRzMX(#aTZHTZl zY<(XLS?8fpbsiQ`=W(FvJO(742Sd@>BZAK3K+kzl@$vW-%W@X!%jLyN9J65h-qn2jxV5dv znLJ3NqNGV{?qIbs2&l85`{GXjrE&}Ew_z+7;(U~z3-gCuRxnrCGWEZhBB&Twdz zv@IBmdrcH3XW* zxyh+e0NDZ`Xzke~5#az=$36KVUibO}3@?14b$#|Jqo@&35Eu28lH`EBUrf{gJNNl1bmatZeUwnV z?X}Jo3m18uEJ-PDg}fI|AOhW8=1pSy2Z1tm-(pya+X5gtvC~?1c|qNYqlb~mG8fN)+pH5>%jSqsW3N#Ppb4O*r zl(L{B6XclpvoUmt;%2QMV7-R=<^rEX=n>}1F68B`sZmer8s0cfv@T$}0J#HZOd^^@ zB}OWmj)$(Vmw$ygmRw;j7bHzKPD6_&AB1_SZ&lz#8-xWRt6^0eTbCfaVacl|EqPgc zteFN-3HwEVBXch)Er1JEfQ%lq?G{h_oIOso3KCDB!PMHEjMa;R|}FRdmh4^;Rbya z>z6X6=zRF1ncvx!T9hdjZWVr@SJ!e#PRUvAb zd0bLz`u#A7_o7opywPCaH@=yxtzF2OSS>VCY zq;prkJws*O|Z`Py2SF{=Swji1lCF*ydKv4WPBl zGz%zYa;RMl7sEp@nYy>aEI+Sr#>}cc4LcFQ!mGU^mqir^)e-3w(CFA^A%LsgW+WH^ z%@%HIP5cPxpsHF4X98*(ln}*Tij)<#^tw?9_OO1ag5lPo7n|w9Wo~G$6}Ti`2!Vhy zeD=?#g$WAlB6zvuaj*+9`5>UP8KYqA)16g+g|xUn8f#u&P~96u`TPqrN@z%qCKo7u zm}G-0Htx_JepU+lsZ;pSSGZ95Utt z1ZiA>oRL`xJVBp5Ri+`Ky2KU!T(IaC#A<6xc0FmfG^0H6%TFcC9$O7&;g@a7 zz#%eOI$!AcvY_+zMd+7yZs=T3_dH>FfyhXCg&0M6O6NA$D`r_Dmn@!;S009xOYqjM zd^ELzKQ=9tttG4l5Acvc9RS^bd!D49rc)|Lj$uH@fcVd7jCtc`kaTW2l9x=WNJjI* z9w2=4{k!ki`|8-dLMGO;*R#Uwsl1ttkCorTde5!K)euqj2KzvP@y2xnqFqtYXkny% z{DvWdy58KvmKcRP3eA+bV5G=-mN4=+2T3quvl)wuzoT|iffGQKuY~~koBh|{n>Mvi zCd3}vPIRaMe*vx3Zf<5~jveTAfuT>x1!V7{Ne0VW(j=+9+>t5;0=o~)TL+4#IU{f7 zwcd#`VIy({33k;daJjve7oS*E7Om2BFK_8UQwm+N?B%`g?#l{8U5z;dFFAOy>ADq=9T zA~iSWQJK)(7NTt}Xwkbx6-8caK_+fh9z*x)X0^dpDIsGymk7^mychnlIKcuzhH_hb zku%GH+^HY=0E)G^W^+@XdK}!H&k6ZvX#oJ*RRhE!aFIG@jOOr|U8;`!Oy-dPdJ;=i z=E2-&8w3FqP zo?;~)B1>Aqro?#7QkW3mm|^-79s^OPQGiyH}(&(lbqv3|+AY9S?Ac_vb5*=Wg3dau}?o8h^X6U}qLI@2{d0I6!u|m*gX_ff;Cluldj?uh!zRDsv;z@5hX?Wo z?(dKzePE={17mbP1UhE`1qQq_9E-Esg6}$_P(o?rO|3O69Ti5ntpo;z*pLZW7Ezs1 zh{z8rcWe8obGSOWQ|5ADn6Ym|wNxX+&1631aSSRM4nQAF zC_lThk(&N%+EP0yul%xdv^#`{Rkh1NF|;@NolBuZ+y*z))hXJI(JR&3+SFDmjWu%; zWdg}Uh)D@L3)*|&P#bgUTp#nIzQL1!`?q%6m<<>qQ&u#Fi|CQpSL#w+$+)h4cYp|z z6Z?;;Gi#l;tzeHN5O6>AUwj)}dMW*Mear5*=Q+^KO4XQ{&2t8ehQ|nwl?A62?2hT} zjkz-J;SSymE{l(va^DL@3$A8dul#?1CEv=zIwn4jjP1}b-IQr9ABxLV)r3395{nGi zO#(F|ge7_OmshI4y`AQXoYKS=%xRy|&Dy(eA(=a2@5HivfOZBBk=xwJWZ zRii5nz*$KcxaX-LZ@74cpSMrpwLY?#wKjrLk*~(TO2J5!l&WC)$dLv5OEn=p$9o(m z;*fwl%VpONyft`HMZP@y^6Yv0fx0v?X%SUy)UY#pOwZfr_8ZYNd2asurTOny*~_$1lcuQ@ka5KB_a)hzjI@nsFN%hwBuF0=y`aDTAf>^1roT*!WvaWG7OX z7M`EL)K1Agm}^sdi?*f7%X_+XjKaz%bAIP)B4#;@Slr1bz&{P=p4$K`>}=Ty;CERZBOI@R zZG|hz`Mz9{Z1NnA$TRB&=_xShc^ba@a$`?J>dyPtSAlEEb7j*8uC~g6dUmykdjT_X zjMA~?TO$HjoUNK^yJg`@#?A;&j2r-dAw`brY~%?1L{2#|2<25yxE`w|@{Hwrv%2oN zb73HOx0o&DUV|BB4z8>`eBQt@kq#FWf_IJMPYREVbPV6XO_vEc?)v93cxNm}$jLpz zhuNTn)L?N6E7BOJfHaCjFOgl&suL)9GP3Dy*7q`H3kv1$q5hmtd{9q59F8{-f740bJ8vAVwTo^)c%` zn~)zeOx>n=M>Oe}{6h+vzd%V+0LHevI9i8i7*quTzjCd0@X;~31kL_MU+Yf)PZBe( z{!ZIj82A(1?d9`uVB8!niFNCx{uWk;WU!%K+L~`n7$@wGCC)NnOImo=0Nt`Cm`gny z#ic-s>9Q}(61r(51d{s=#2oC7*!}i8NnLktW2RQgx3z~|ku_3|W{5epP#B0 z2&2&gmiWfu4g(`IK7+l#u(ep&fk>)qH;Wf(_c47XF{w8zFF;*o$IY*G3wf40xGr-cV z^49jjvdJ7A+bg7XR4+lXkg5=6M zHIOwK!qAwwU`a-D7MD>UWQyug1qeC!HE@WhcWm>hw#P?U=|hBF>3jw!8&TC}gx#@x zNiwMrSPl3;Rf^9Ac(!LYstgn{n4i&QT01y0O3o2jPGk^Ie^_(+S7*<=%;}qG+gI>E zX9w@KhhNT`&aRtl?G%NW%De2hFbl9~e|73v(!VRiPr)!#RhkY%BTK`09`E0b$hporCqu2gHJN?7y?QZf;vIEzGH z`+*T}X=B!A)BWSndgn|)&PVKu8EPFw&TQ@2>x@Na*EbHh_lD}ZuqjFPCs+=IQ1ce> z*%pT@t>&~zU_EqqMT^C@B9LywjMJL2O@6Gtp`H3`;Fu~pPBA`yLdcsgh-WRMm?p{b zi&iMUUeOQdo6C&Ofo!*(_G)`okUvU(sW~w@z}&%G7LyHN%Vfu#3+b<-arPT-PM3t{ zvJ`y*3@)eX&B{REdf37}%DIc9gQylR$n(kbKA*um7L~c;ckCUb@ec+{*4(O>vnwn2 zNDvk;P`h6rq;o64XaV)Wv4Ph9O^R9U<@f2Ix4MS=EnOYZ2K6Kwy;?A2n-jvwTZ2tt z&+^8j$jw4-?JPA3DZ0I}6-SG_WF!3OKIaAV@~2K_G*PRI-{EKX^D5z|-7}s_nXr~D z$$XWFww?PrRaWUu_X}p~G1+x7XSd>2STV1i^4xfnWsgD4KNYY@BR{9nI6_lK1q6$W z*c}9K4+eCB+lws3KhRdp<;omD2nK>X$+-Jc8n+Af-h8WNn^YFV)qbjjztYq2UPeZN z){U2$fvvYiX&uwd^YDle}tPa(&q_uH2kH-zC-HqHk%zB0^Gfp_L*P^Sfpz!;y z6jLe?2FD3Wl`K<8XO-}9978O>?fa`RzRPxYDvHX_71n`vgvuyG_NC9`)NQ z7meSFW}DHbIyDwi-dwj0Qfo~wTWYjn%B=0HS1oPmXXlD)J@rve)hl=9a!fa*Su{ zrU$loNYWX$W*w$8o-oA00A^4xFM>O;lqI>R=!gau)SWbW!?PFULa5SK)^6&iNzR0A zGdP%H-hA<$sb<*@B+*pu|!~hcIQ=JweW}4(!%W>$u!Gl$~~+sI0+qH1*JBE3%JY1 zLk1F}YtYi7_kq}!3NL3wV%EsWB(h-S16QsAO?RdT&45EG?X|Rb2|0_|-I(~A+^u5tq{JGK^A&NygD+Y-K)lck& z>Y(!Rt2%OB=OmFxj9rnO8%_poYZ7RVqN>lZhrs}!8a6-s3p+hzr|>c9ICp_V>ObGr z0=K_n$GGVkk65;tg{4UAS6^S(O|>sswji&sorqqY+)f}u7`8&g?y({XpRtbWe?c&l zQt(3N;qa1uSX8Dxz4!0zPLVg>r<6o8Nmyj<_^=fAXG0Z36DF%Bw|@cE&?tdtxIJg@ zF3w-QxtOH!9~Sa{T*z^$*qC>LiUr`b1tNIu6`m~XaAlG;Sq!{%T$zL_n!F(I75Xcj zADzYEx$0J`atbaGS5|&jCJF9ekux^(tz&18SF))s#G@2*4nyigF1Rjfh78<@s>S+N zT-QEF>u0gh8*|H+1OOUmGS{S=*GY{P5v;;lsaQV2<{!!G_Fb+p)_1J4uJ`b@eaB`^ zOKU!Mo^dM{(y~}!W*YEhBZM@rHNt2<_~M2_>UPBlR?7>sdqIx+iZTs_S@mr_X3+!i zVe7%22)kj!n(#iwQ~*;UZAzotdz!~kVj`mjHyUSeX?^0#eHbh=1IKFe@=Pr{eG-=$ zGdWViAhQ`{5X};4Z+(5;QL06oh~-+MO_t<7m;Lo2t>@}Ij*aiVRBTozxHxJ;XvR%6 z{s@c{BU1AC>)uue+Lz^>*1p*N*}cqfxtNdjQjCrJ{!CMtI{O3sCvZNma+#5jy31+F zlbGc%3<-WTNz<Mwu& zbKimw5^Q?mS%;%v>PNHiT`@!5mOew$7khGhx#TpK3g6MYS7UQDPCJm)$m6*}`jw#` z$+}C?AB9*$25zZC!6f&8Woj4DO#ne^iJA~hxLo_I4riXaEChK=# zsy~_XqPpD_D-b2y6poUDL?$H%WJnL)JW%!&L`H?hJ941f&z?PdhRC-6`JewuWKJlF zS;W)z7vZh{c-{e~sh|IK{^olKW6uhOfce;~{&B5gH6s7x`iHj_@O%ARFnPIU>F(3F z8$bK=pZ?VIHqQZOgByf08(D8NKLFhqZb>VZV_H|y39?G;O`G!hqA=J|ntl@=6?og> zog)p98s;pR60%0K=<(*nY(x@%%g9W9o69n*s&AM?2LsTI9aS{dSwdjfL8HA+n;{1V z8e36M3{+B8b0VYMjK`H8~PX5xd8k=sC<FxnP*o>;P{e}@&Su~oy6;ezo)>_8=$)47MFT+5oEtKJ9 z=|F~=NhdV#+Ffyr+wIcw(GOTrnytWO%~AlmD>q4|`7Mhb|C~T3pz1A|N^Idceupmk zlrC$q33>p>lgTkM1DRt!DV|>a`7W#Ly`h=HB3D|!fl_f2P}!;^K5ACuQIxbZBQ!L& z)6!~b&lTG+u%N8(-&cY&+r3cV-aux3}ZyzH**V%#nCg52?U)B!&>oZ(5aG1 zSzzNse(rN-7?>?=*2j)l8}c|g+>6L|!&(VuWxK1*mo=vZua4+sYL=34*Ey?O!&-njKK+l%Yh`FJ}LfI8bT zLUWeOJ04q^*=PS;k&G%2$$%;)BM#QSYIi#OK=m7>?s^e@8%g1No+XqR?~EM_T5cg@ zc2!Ojt`^t3_7*m^jUmfxF>`HR!ya#>P#gqaeVJ*=9gsvN?1XLEKN=E5f`l1tsF&Am zIVt$Ot;TQgLA0u&KYcH<+UF5Xk{;!&F2&fp0ks!fyAcVBYg)4_0s)37<=EQZ0Up|w zg;ZV@Cn@Y|kF?BVf8gQ!0E{Eh@{_;y$=~|qZ~d?4Z{1snP6b0rJxswm5|tG=LWGu{ zWP>dwWa}tCFH9NT$6vXAb-*AQGiC0-v04AfaWW*1T6^Uh3 zrc4yomR3P>Xtvz4T6{oKYL~p#+e0kxh34qzSFKt7^^E@NeP4C49~1pKokAfc%p2`+ ze6M@SAFnsO$t+(;eMxkByDO9X;MSj<>rc-0=I3|mT)#x`xxO)>d_Ppn^q7hT2<={+ zp^nX06+1%(XEmSR)3^Er8fngD<&5xRLM;F=h?6^VkwRNNT5Q9dkE`@rtm1+Z0$vzG zCVRXB_<&FYl|!+(;>eUd8=1TakW(`np*C6a?s?;*&z-wMJ7IV5312N}-3jp_Z?_ci zFpH$31xqRX^R-^Q7GTt-hby8bw}K(k9LrO!0F7#q-A8wQpl$XU;JhZ5YPt}O>rZV7 zu4?W!O)+pmX(9@NBC4J$6a&sSuIwJB8|LI9AV|%EcQH^Y!Dlno1fP8c8*Cb6Fmkr} zx)2f2)ImU_7-0tE>%X4|aBGYI z_0*^zRG>hWUt4*D2`@JFwPdOkf#lSU~>hWRXbAmscL|Hj9O-xdj%6vI; z&FC*{?V2|bZ9XdVoQWb?dIHgox3YSuSpE<|iv+9C4d7w*-vkeSWecvIsnJ8QN5*SNJ8=0e zKDi_O6?yb|pT`29j1~pU#S8MkPd@zdxAE!MCnrBY8~^p6e?0jxfq#5)`t|8=_K!cF zo}Qfi{N0;xZ>}!>$xna#S(NE5{`K3*&+Ou#fsLG=e*N!3!>dVSTohxOk8$Z=0LE`X zBX)r{IWN^6RH4jU33?BtK1BSr19~jdZ)kklXAphXI|6WYz2aME%iDJ9nMnmpGnvzT zNn&I1T3YLCBP!V5>Jq^4V&29WgJ8#4%*%9VpkvUQzgqWDF~C$Ju;Di5HCuXyMz^RE zx#jG$2G;Q3kB&?LM9QdeR%8ftkuf_~?p^FQ*r_|#Ucs|_ww43~ z8}VP+_gY%&bV2o9$>A4q@bx<=dE8FIMlcl;lu-lPH-oa-i6gI?&;*2aVb5`hb5N|E zQ zpGOK+VCoJ+63t%*iUpBhhE2YH8(v}}DOn^&F7oxV1aM=+LJAOk0OYHzblBnX5EyUO zqM57$(e)*YtU^*sYYHNYlgb2fZjV&y7-9v6E83U%ZqbM;v}u^qKS;p)_~wz6uoQZ) z{1a_%M9|WWWFz#%HAi=)0AER=( zLZFRXA5TWr`moQHNz~r5wP#bRQ#)!*qsM)P+KE7_q-CvM``cQtl-@-QN(m_8wQ*9l zbsj`W9gqipw-<|#B~$$i(5*;?Jfh%Gh>C}nx_O;qp~N&_zKqGf7lJOTc&CQhEoYhV z#Pehy`Ui~a@F34+#cLU1ot-;LP&%{x*N`wprBckw>lf-AZxP$2U|tg2jTDY(gO@v> zwztZBe@2w|dx#k!k-6PI%U+nckh+YXK_7a}4zNHc35bwbW4Ld!H>=)_IH&Hin4DlA z@M;HSdN`y3Q6ONMt+}YcKqVqpW!2q7;BMF@p?XzO3H@{?`x008hv!t zuQIl{Z+MUnzBok@X;edPki#Lq4V|KO-uixb^!FArbG1jC>A9QY5C@sZx>~cvOluf9 zE6`5pW#j9WjfdmwYG=gn!P4jpHy_n2kdLqWvwsH99(A2hckGqZXl^jc_H(u34!K`A zo{sELQcxN__70*i&FPu$!ZEe~afNXfZ&RPJrc=XiN$^ipx87sHqIKQ*W7;6zZ%~Dq*=X)v7uR zft`T0WP2G}LGrAz!KOl>h%`k89Ybj*IxEi3Ag9$`oflwfuV4d5WTCu>u4t`NlC+b@ zgm5zUf=uC4?E|HtDm08>g1Nv@76%SwbipUAc>UF2E(kO(#1xT!qbmmuHUal|0))w4L^_H ziWa;C@$D7vb2<*Fcyu*-E=O@^Kko?L;4}*^b(6f9DTMF>;B1!_|>sN=`l zh^JzBzpBCzOb~4#esBW#4?I|U-7L3+ak9WL2lnCps^IO+rB=@t*|OA=UM&?e+95!R z^ex?C3U)yCF*2jmbkNQ0u6MB+*$OB<&8EfO>uE)f?G)i~(iE1bv`Gq?&hiPPitBJU zvQ}eM#TSt}l_8+Amq$Q=|B*Ko6k5F^U8x$R!!4g!1p>gPfU|esEQmGp9fm8PPh(@f zO?_Xm>mC9qTGO~XtN@ixpd>s6k2~MNNTbac`mF0IW~x60VAA({rhk3pqvLr%0=2{n zC5YeIz;KbF-9D8N9{UBYeg&SR=!VOe3jX;|Lg+1f6pC~`Z8KhguU z01_@6AEr>TW10#`p{Z%Gx|oC*}u*#6U$#`77?3A-J_K9O0WsNFgbl^NVq$bGrqeh zs2H*R1kIi90;lQ`2K{Z}d%2@JXaMu!9%huhtI@yyAfRxXRe^pEmWmFC#vX8)oU~%Iv;_Aig|bHGV2|T`w1IA!NfaaEwI5Zp$8_$AzJ4S2u{O zXSOzzekU86Xdp^t7DK;MRgAnsT-po{?cXOC*DHlm-xp-}k*<{4*>vK2-S~Z7&JBel zN+x^w0?)93)EWzo2KBB~JUg7T2e+97;h&aBA+Vz^=IQ7e(BeHO6B>Wgnq%T=lN8Jf zU(fcnO?+RX9wR70KWpCD?FJm_QR9FZ)ZLJDm$hyWoCVBAlHFM_&R=Y}{5pQgE zURwLut2O9NW_}Qrkk4dal0C)3BIzh z++Hr)1I{Q?o@Pbsyc|#t9a+!Ppjt1G(*As&;q%}YexLlyOIa!~zP_t2U{+3MdTBe0 z{ojc`lfz@}1q_6pr9~B)18}bHQSK%=+6H@)?r?Zu<_iQ=L6|_DQyb>RkJFFO#~G|6w|asb*o1b_=$|G0A^}4nTUrhVtJaLng9+Bm^)SRVcgQ@6Sn@slyrFo zNmhKwX|A_yjgeQgSltYNvRe-cJv8pZ5O_-O6lHCj3*TszUl9S3`EoP-1Ga|EsLxO^0=1cxz0qe(L%_k3o_)